| //===-- RISCVFrameLowering.cpp - RISCV Frame Information ------------------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file contains the RISCV implementation of TargetFrameLowering class. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "RISCVFrameLowering.h" |
| #include "RISCVMachineFunctionInfo.h" |
| #include "RISCVSubtarget.h" |
| #include "llvm/BinaryFormat/Dwarf.h" |
| #include "llvm/CodeGen/MachineFrameInfo.h" |
| #include "llvm/CodeGen/MachineFunction.h" |
| #include "llvm/CodeGen/MachineInstrBuilder.h" |
| #include "llvm/CodeGen/MachineRegisterInfo.h" |
| #include "llvm/CodeGen/RegisterScavenging.h" |
| #include "llvm/IR/DiagnosticInfo.h" |
| #include "llvm/MC/MCDwarf.h" |
| #include "llvm/Support/LEB128.h" |
| |
| #include <algorithm> |
| |
| using namespace llvm; |
| |
| // For now we use x18, a.k.a s2, as pointer to shadow call stack. |
| // User should explicitly set -ffixed-x18 and not use x18 in their asm. |
| static void emitSCSPrologue(MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MI, |
| const DebugLoc &DL) { |
| if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) |
| return; |
| |
| const auto &STI = MF.getSubtarget<RISCVSubtarget>(); |
| Register RAReg = STI.getRegisterInfo()->getRARegister(); |
| |
| // Do not save RA to the SCS if it's not saved to the regular stack, |
| // i.e. RA is not at risk of being overwritten. |
| std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
| if (llvm::none_of( |
| CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; })) |
| return; |
| |
| Register SCSPReg = RISCVABI::getSCSPReg(); |
| |
| auto &Ctx = MF.getFunction().getContext(); |
| if (!STI.isRegisterReservedByUser(SCSPReg)) { |
| Ctx.diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."}); |
| return; |
| } |
| |
| const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| if (RVFI->useSaveRestoreLibCalls(MF)) { |
| Ctx.diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), |
| "Shadow Call Stack cannot be combined with Save/Restore LibCalls."}); |
| return; |
| } |
| |
| const RISCVInstrInfo *TII = STI.getInstrInfo(); |
| bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit); |
| int64_t SlotSize = STI.getXLen() / 8; |
| // Store return address to shadow call stack |
| // s[w|d] ra, 0(s2) |
| // addi s2, s2, [4|8] |
| BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::SD : RISCV::SW)) |
| .addReg(RAReg) |
| .addReg(SCSPReg) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI)) |
| .addReg(SCSPReg, RegState::Define) |
| .addReg(SCSPReg) |
| .addImm(SlotSize) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| static void emitSCSEpilogue(MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MI, |
| const DebugLoc &DL) { |
| if (!MF.getFunction().hasFnAttribute(Attribute::ShadowCallStack)) |
| return; |
| |
| const auto &STI = MF.getSubtarget<RISCVSubtarget>(); |
| Register RAReg = STI.getRegisterInfo()->getRARegister(); |
| |
| // See emitSCSPrologue() above. |
| std::vector<CalleeSavedInfo> &CSI = MF.getFrameInfo().getCalleeSavedInfo(); |
| if (llvm::none_of( |
| CSI, [&](CalleeSavedInfo &CSR) { return CSR.getReg() == RAReg; })) |
| return; |
| |
| Register SCSPReg = RISCVABI::getSCSPReg(); |
| |
| auto &Ctx = MF.getFunction().getContext(); |
| if (!STI.isRegisterReservedByUser(SCSPReg)) { |
| Ctx.diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), "x18 not reserved by user for Shadow Call Stack."}); |
| return; |
| } |
| |
| const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| if (RVFI->useSaveRestoreLibCalls(MF)) { |
| Ctx.diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), |
| "Shadow Call Stack cannot be combined with Save/Restore LibCalls."}); |
| return; |
| } |
| |
| const RISCVInstrInfo *TII = STI.getInstrInfo(); |
| bool IsRV64 = STI.hasFeature(RISCV::Feature64Bit); |
| int64_t SlotSize = STI.getXLen() / 8; |
| // Load return address from shadow call stack |
| // l[w|d] ra, -[4|8](s2) |
| // addi s2, s2, -[4|8] |
| BuildMI(MBB, MI, DL, TII->get(IsRV64 ? RISCV::LD : RISCV::LW)) |
| .addReg(RAReg, RegState::Define) |
| .addReg(SCSPReg) |
| .addImm(-SlotSize) |
| .setMIFlag(MachineInstr::FrameDestroy); |
| BuildMI(MBB, MI, DL, TII->get(RISCV::ADDI)) |
| .addReg(SCSPReg, RegState::Define) |
| .addReg(SCSPReg) |
| .addImm(-SlotSize) |
| .setMIFlag(MachineInstr::FrameDestroy); |
| } |
| |
| // Get the ID of the libcall used for spilling and restoring callee saved |
| // registers. The ID is representative of the number of registers saved or |
| // restored by the libcall, except it is zero-indexed - ID 0 corresponds to a |
| // single register. |
| static int getLibCallID(const MachineFunction &MF, |
| const std::vector<CalleeSavedInfo> &CSI) { |
| const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| |
| if (CSI.empty() || !RVFI->useSaveRestoreLibCalls(MF)) |
| return -1; |
| |
| Register MaxReg = RISCV::NoRegister; |
| for (auto &CS : CSI) |
| // RISCVRegisterInfo::hasReservedSpillSlot assigns negative frame indexes to |
| // registers which can be saved by libcall. |
| if (CS.getFrameIdx() < 0) |
| MaxReg = std::max(MaxReg.id(), CS.getReg().id()); |
| |
| if (MaxReg == RISCV::NoRegister) |
| return -1; |
| |
| switch (MaxReg) { |
| default: |
| llvm_unreachable("Something has gone wrong!"); |
| case /*s11*/ RISCV::X27: return 12; |
| case /*s10*/ RISCV::X26: return 11; |
| case /*s9*/ RISCV::X25: return 10; |
| case /*s8*/ RISCV::X24: return 9; |
| case /*s7*/ RISCV::X23: return 8; |
| case /*s6*/ RISCV::X22: return 7; |
| case /*s5*/ RISCV::X21: return 6; |
| case /*s4*/ RISCV::X20: return 5; |
| case /*s3*/ RISCV::X19: return 4; |
| case /*s2*/ RISCV::X18: return 3; |
| case /*s1*/ RISCV::X9: return 2; |
| case /*s0*/ RISCV::X8: return 1; |
| case /*ra*/ RISCV::X1: return 0; |
| } |
| } |
| |
| // Get the name of the libcall used for spilling callee saved registers. |
| // If this function will not use save/restore libcalls, then return a nullptr. |
| static const char * |
| getSpillLibCallName(const MachineFunction &MF, |
| const std::vector<CalleeSavedInfo> &CSI) { |
| static const char *const SpillLibCalls[] = { |
| "__riscv_save_0", |
| "__riscv_save_1", |
| "__riscv_save_2", |
| "__riscv_save_3", |
| "__riscv_save_4", |
| "__riscv_save_5", |
| "__riscv_save_6", |
| "__riscv_save_7", |
| "__riscv_save_8", |
| "__riscv_save_9", |
| "__riscv_save_10", |
| "__riscv_save_11", |
| "__riscv_save_12" |
| }; |
| |
| int LibCallID = getLibCallID(MF, CSI); |
| if (LibCallID == -1) |
| return nullptr; |
| return SpillLibCalls[LibCallID]; |
| } |
| |
| // Get the name of the libcall used for restoring callee saved registers. |
| // If this function will not use save/restore libcalls, then return a nullptr. |
| static const char * |
| getRestoreLibCallName(const MachineFunction &MF, |
| const std::vector<CalleeSavedInfo> &CSI) { |
| static const char *const RestoreLibCalls[] = { |
| "__riscv_restore_0", |
| "__riscv_restore_1", |
| "__riscv_restore_2", |
| "__riscv_restore_3", |
| "__riscv_restore_4", |
| "__riscv_restore_5", |
| "__riscv_restore_6", |
| "__riscv_restore_7", |
| "__riscv_restore_8", |
| "__riscv_restore_9", |
| "__riscv_restore_10", |
| "__riscv_restore_11", |
| "__riscv_restore_12" |
| }; |
| |
| int LibCallID = getLibCallID(MF, CSI); |
| if (LibCallID == -1) |
| return nullptr; |
| return RestoreLibCalls[LibCallID]; |
| } |
| |
| // Return true if the specified function should have a dedicated frame |
| // pointer register. This is true if frame pointer elimination is |
| // disabled, if it needs dynamic stack realignment, if the function has |
| // variable sized allocas, or if the frame address is taken. |
| bool RISCVFrameLowering::hasFP(const MachineFunction &MF) const { |
| const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); |
| |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| return MF.getTarget().Options.DisableFramePointerElim(MF) || |
| RegInfo->hasStackRealignment(MF) || MFI.hasVarSizedObjects() || |
| MFI.isFrameAddressTaken(); |
| } |
| |
| bool RISCVFrameLowering::hasBP(const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const TargetRegisterInfo *TRI = STI.getRegisterInfo(); |
| |
| // If we do not reserve stack space for outgoing arguments in prologue, |
| // we will adjust the stack pointer before call instruction. After the |
| // adjustment, we can not use SP to access the stack objects for the |
| // arguments. Instead, use BP to access these stack objects. |
| return (MFI.hasVarSizedObjects() || |
| (!hasReservedCallFrame(MF) && (!MFI.isMaxCallFrameSizeComputed() || |
| MFI.getMaxCallFrameSize() != 0))) && |
| TRI->hasStackRealignment(MF); |
| } |
| |
| // Determines the size of the frame and maximum call frame size. |
| void RISCVFrameLowering::determineFrameLayout(MachineFunction &MF) const { |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| |
| // Get the number of bytes to allocate from the FrameInfo. |
| uint64_t FrameSize = MFI.getStackSize(); |
| |
| // Get the alignment. |
| Align StackAlign = getStackAlign(); |
| |
| // Make sure the frame is aligned. |
| FrameSize = alignTo(FrameSize, StackAlign); |
| |
| // Update frame info. |
| MFI.setStackSize(FrameSize); |
| |
| // When using SP or BP to access stack objects, we may require extra padding |
| // to ensure the bottom of the RVV stack is correctly aligned within the main |
| // stack. We calculate this as the amount required to align the scalar local |
| // variable section up to the RVV alignment. |
| const TargetRegisterInfo *TRI = STI.getRegisterInfo(); |
| if (RVFI->getRVVStackSize() && (!hasFP(MF) || TRI->hasStackRealignment(MF))) { |
| int ScalarLocalVarSize = FrameSize - RVFI->getCalleeSavedStackSize() - |
| RVFI->getVarArgsSaveSize(); |
| if (auto RVVPadding = |
| offsetToAlignment(ScalarLocalVarSize, RVFI->getRVVStackAlign())) |
| RVFI->setRVVPadding(RVVPadding); |
| } |
| } |
| |
| // Returns the stack size including RVV padding (when required), rounded back |
| // up to the required stack alignment. |
| uint64_t RISCVFrameLowering::getStackSizeWithRVVPadding( |
| const MachineFunction &MF) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| return alignTo(MFI.getStackSize() + RVFI->getRVVPadding(), getStackAlign()); |
| } |
| |
| // Returns the register used to hold the frame pointer. |
| static Register getFPReg(const RISCVSubtarget &STI) { return RISCV::X8; } |
| |
| // Returns the register used to hold the stack pointer. |
| static Register getSPReg(const RISCVSubtarget &STI) { return RISCV::X2; } |
| |
| static SmallVector<CalleeSavedInfo, 8> |
| getNonLibcallCSI(const MachineFunction &MF, |
| const std::vector<CalleeSavedInfo> &CSI) { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| SmallVector<CalleeSavedInfo, 8> NonLibcallCSI; |
| |
| for (auto &CS : CSI) { |
| int FI = CS.getFrameIdx(); |
| if (FI >= 0 && MFI.getStackID(FI) == TargetStackID::Default) |
| NonLibcallCSI.push_back(CS); |
| } |
| |
| return NonLibcallCSI; |
| } |
| |
| void RISCVFrameLowering::adjustStackForRVV(MachineFunction &MF, |
| MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MBBI, |
| const DebugLoc &DL, int64_t Amount, |
| MachineInstr::MIFlag Flag) const { |
| assert(Amount != 0 && "Did not need to adjust stack pointer for RVV."); |
| |
| const Register SPReg = getSPReg(STI); |
| |
| // Optimize compile time offset case |
| StackOffset Offset = StackOffset::getScalable(Amount); |
| if (STI.getRealMinVLen() == STI.getRealMaxVLen()) { |
| // 1. Multiply the number of v-slots by the (constant) length of register |
| const int64_t VLENB = STI.getRealMinVLen() / 8; |
| assert(Amount % 8 == 0 && |
| "Reserve the stack by the multiple of one vector size."); |
| const int64_t NumOfVReg = Amount / 8; |
| const int64_t FixedOffset = NumOfVReg * VLENB; |
| if (!isInt<32>(FixedOffset)) { |
| report_fatal_error( |
| "Frame size outside of the signed 32-bit range not supported"); |
| } |
| Offset = StackOffset::getFixed(FixedOffset); |
| } |
| |
| const RISCVRegisterInfo &RI = *STI.getRegisterInfo(); |
| // We must keep the stack pointer aligned through any intermediate |
| // updates. |
| RI.adjustReg(MBB, MBBI, DL, SPReg, SPReg, Offset, |
| Flag, getStackAlign()); |
| } |
| |
| static MCCFIInstruction createDefCFAExpression(const TargetRegisterInfo &TRI, |
| Register Reg, |
| uint64_t FixedOffset, |
| uint64_t ScalableOffset) { |
| assert(ScalableOffset != 0 && "Did not need to adjust CFA for RVV"); |
| SmallString<64> Expr; |
| std::string CommentBuffer; |
| llvm::raw_string_ostream Comment(CommentBuffer); |
| // Build up the expression (Reg + FixedOffset + ScalableOffset * VLENB). |
| unsigned DwarfReg = TRI.getDwarfRegNum(Reg, true); |
| Expr.push_back((uint8_t)(dwarf::DW_OP_breg0 + DwarfReg)); |
| Expr.push_back(0); |
| if (Reg == RISCV::X2) |
| Comment << "sp"; |
| else |
| Comment << printReg(Reg, &TRI); |
| |
| uint8_t buffer[16]; |
| if (FixedOffset) { |
| Expr.push_back(dwarf::DW_OP_consts); |
| Expr.append(buffer, buffer + encodeSLEB128(FixedOffset, buffer)); |
| Expr.push_back((uint8_t)dwarf::DW_OP_plus); |
| Comment << " + " << FixedOffset; |
| } |
| |
| Expr.push_back((uint8_t)dwarf::DW_OP_consts); |
| Expr.append(buffer, buffer + encodeSLEB128(ScalableOffset, buffer)); |
| |
| unsigned DwarfVlenb = TRI.getDwarfRegNum(RISCV::VLENB, true); |
| Expr.push_back((uint8_t)dwarf::DW_OP_bregx); |
| Expr.append(buffer, buffer + encodeULEB128(DwarfVlenb, buffer)); |
| Expr.push_back(0); |
| |
| Expr.push_back((uint8_t)dwarf::DW_OP_mul); |
| Expr.push_back((uint8_t)dwarf::DW_OP_plus); |
| |
| Comment << " + " << ScalableOffset << " * vlenb"; |
| |
| SmallString<64> DefCfaExpr; |
| DefCfaExpr.push_back(dwarf::DW_CFA_def_cfa_expression); |
| DefCfaExpr.append(buffer, buffer + encodeULEB128(Expr.size(), buffer)); |
| DefCfaExpr.append(Expr.str()); |
| |
| return MCCFIInstruction::createEscape(nullptr, DefCfaExpr.str(), |
| Comment.str()); |
| } |
| |
| void RISCVFrameLowering::emitPrologue(MachineFunction &MF, |
| MachineBasicBlock &MBB) const { |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
| const RISCVInstrInfo *TII = STI.getInstrInfo(); |
| MachineBasicBlock::iterator MBBI = MBB.begin(); |
| |
| Register FPReg = getFPReg(STI); |
| Register SPReg = getSPReg(STI); |
| Register BPReg = RISCVABI::getBPReg(); |
| |
| // Debug location must be unknown since the first debug location is used |
| // to determine the end of the prologue. |
| DebugLoc DL; |
| |
| // All calls are tail calls in GHC calling conv, and functions have no |
| // prologue/epilogue. |
| if (MF.getFunction().getCallingConv() == CallingConv::GHC) |
| return; |
| |
| // Emit prologue for shadow call stack. |
| emitSCSPrologue(MF, MBB, MBBI, DL); |
| |
| // Since spillCalleeSavedRegisters may have inserted a libcall, skip past |
| // any instructions marked as FrameSetup |
| while (MBBI != MBB.end() && MBBI->getFlag(MachineInstr::FrameSetup)) |
| ++MBBI; |
| |
| // Determine the correct frame layout |
| determineFrameLayout(MF); |
| |
| // If libcalls are used to spill and restore callee-saved registers, the frame |
| // has two sections; the opaque section managed by the libcalls, and the |
| // section managed by MachineFrameInfo which can also hold callee saved |
| // registers in fixed stack slots, both of which have negative frame indices. |
| // This gets even more complicated when incoming arguments are passed via the |
| // stack, as these too have negative frame indices. An example is detailed |
| // below: |
| // |
| // | incoming arg | <- FI[-3] |
| // | libcallspill | |
| // | calleespill | <- FI[-2] |
| // | calleespill | <- FI[-1] |
| // | this_frame | <- FI[0] |
| // |
| // For negative frame indices, the offset from the frame pointer will differ |
| // depending on which of these groups the frame index applies to. |
| // The following calculates the correct offset knowing the number of callee |
| // saved registers spilt by the two methods. |
| if (int LibCallRegs = getLibCallID(MF, MFI.getCalleeSavedInfo()) + 1) { |
| // Calculate the size of the frame managed by the libcall. The libcalls are |
| // implemented such that the stack will always be 16 byte aligned. |
| unsigned LibCallFrameSize = alignTo((STI.getXLen() / 8) * LibCallRegs, 16); |
| RVFI->setLibCallStackSize(LibCallFrameSize); |
| } |
| |
| // FIXME (note copied from Lanai): This appears to be overallocating. Needs |
| // investigation. Get the number of bytes to allocate from the FrameInfo. |
| uint64_t StackSize = getStackSizeWithRVVPadding(MF); |
| uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize(); |
| uint64_t RVVStackSize = RVFI->getRVVStackSize(); |
| |
| // Early exit if there is no need to allocate on the stack |
| if (RealStackSize == 0 && !MFI.adjustsStack() && RVVStackSize == 0) |
| return; |
| |
| // If the stack pointer has been marked as reserved, then produce an error if |
| // the frame requires stack allocation |
| if (STI.isRegisterReservedByUser(SPReg)) |
| MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), "Stack pointer required, but has been reserved."}); |
| |
| uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
| // Split the SP adjustment to reduce the offsets of callee saved spill. |
| if (FirstSPAdjustAmount) { |
| StackSize = FirstSPAdjustAmount; |
| RealStackSize = FirstSPAdjustAmount; |
| } |
| |
| // Allocate space on the stack if necessary. |
| RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(-StackSize), |
| MachineInstr::FrameSetup, getStackAlign()); |
| |
| // Emit ".cfi_def_cfa_offset RealStackSize" |
| unsigned CFIIndex = MF.addFrameInst( |
| MCCFIInstruction::cfiDefCfaOffset(nullptr, RealStackSize)); |
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| const auto &CSI = MFI.getCalleeSavedInfo(); |
| |
| // The frame pointer is callee-saved, and code has been generated for us to |
| // save it to the stack. We need to skip over the storing of callee-saved |
| // registers as the frame pointer must be modified after it has been saved |
| // to the stack, not before. |
| // FIXME: assumes exactly one instruction is used to save each callee-saved |
| // register. |
| std::advance(MBBI, getNonLibcallCSI(MF, CSI).size()); |
| |
| // Iterate over list of callee-saved registers and emit .cfi_offset |
| // directives. |
| for (const auto &Entry : CSI) { |
| int FrameIdx = Entry.getFrameIdx(); |
| int64_t Offset; |
| // Offsets for objects with fixed locations (IE: those saved by libcall) are |
| // simply calculated from the frame index. |
| if (FrameIdx < 0) |
| Offset = FrameIdx * (int64_t) STI.getXLen() / 8; |
| else |
| Offset = MFI.getObjectOffset(Entry.getFrameIdx()) - |
| RVFI->getLibCallStackSize(); |
| Register Reg = Entry.getReg(); |
| unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::createOffset( |
| nullptr, RI->getDwarfRegNum(Reg, true), Offset)); |
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Generate new FP. |
| if (hasFP(MF)) { |
| if (STI.isRegisterReservedByUser(FPReg)) |
| MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{ |
| MF.getFunction(), "Frame pointer required, but has been reserved."}); |
| // The frame pointer does need to be reserved from register allocation. |
| assert(MF.getRegInfo().isReserved(FPReg) && "FP not reserved"); |
| |
| RI->adjustReg(MBB, MBBI, DL, FPReg, SPReg, |
| StackOffset::getFixed(RealStackSize - RVFI->getVarArgsSaveSize()), |
| MachineInstr::FrameSetup, getStackAlign()); |
| |
| // Emit ".cfi_def_cfa $fp, RVFI->getVarArgsSaveSize()" |
| unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfa( |
| nullptr, RI->getDwarfRegNum(FPReg, true), RVFI->getVarArgsSaveSize())); |
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| |
| // Emit the second SP adjustment after saving callee saved registers. |
| if (FirstSPAdjustAmount) { |
| uint64_t SecondSPAdjustAmount = |
| getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount; |
| assert(SecondSPAdjustAmount > 0 && |
| "SecondSPAdjustAmount should be greater than zero"); |
| RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, |
| StackOffset::getFixed(-SecondSPAdjustAmount), |
| MachineInstr::FrameSetup, getStackAlign()); |
| |
| // If we are using a frame-pointer, and thus emitted ".cfi_def_cfa fp, 0", |
| // don't emit an sp-based .cfi_def_cfa_offset |
| if (!hasFP(MF)) { |
| // Emit ".cfi_def_cfa_offset StackSize" |
| unsigned CFIIndex = MF.addFrameInst(MCCFIInstruction::cfiDefCfaOffset( |
| nullptr, getStackSizeWithRVVPadding(MF))); |
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| if (RVVStackSize) { |
| adjustStackForRVV(MF, MBB, MBBI, DL, -RVVStackSize, |
| MachineInstr::FrameSetup); |
| if (!hasFP(MF)) { |
| // Emit .cfi_def_cfa_expression "sp + StackSize + RVVStackSize * vlenb". |
| unsigned CFIIndex = MF.addFrameInst(createDefCFAExpression( |
| *RI, SPReg, getStackSizeWithRVVPadding(MF), RVVStackSize / 8)); |
| BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) |
| .addCFIIndex(CFIIndex) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| |
| if (hasFP(MF)) { |
| // Realign Stack |
| const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
| if (RI->hasStackRealignment(MF)) { |
| Align MaxAlignment = MFI.getMaxAlign(); |
| |
| const RISCVInstrInfo *TII = STI.getInstrInfo(); |
| if (isInt<12>(-(int)MaxAlignment.value())) { |
| BuildMI(MBB, MBBI, DL, TII->get(RISCV::ANDI), SPReg) |
| .addReg(SPReg) |
| .addImm(-(int)MaxAlignment.value()) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } else { |
| unsigned ShiftAmount = Log2(MaxAlignment); |
| Register VR = |
| MF.getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); |
| BuildMI(MBB, MBBI, DL, TII->get(RISCV::SRLI), VR) |
| .addReg(SPReg) |
| .addImm(ShiftAmount) |
| .setMIFlag(MachineInstr::FrameSetup); |
| BuildMI(MBB, MBBI, DL, TII->get(RISCV::SLLI), SPReg) |
| .addReg(VR) |
| .addImm(ShiftAmount) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| // FP will be used to restore the frame in the epilogue, so we need |
| // another base register BP to record SP after re-alignment. SP will |
| // track the current stack after allocating variable sized objects. |
| if (hasBP(MF)) { |
| // move BP, SP |
| BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADDI), BPReg) |
| .addReg(SPReg) |
| .addImm(0) |
| .setMIFlag(MachineInstr::FrameSetup); |
| } |
| } |
| } |
| } |
| |
| void RISCVFrameLowering::emitEpilogue(MachineFunction &MF, |
| MachineBasicBlock &MBB) const { |
| const RISCVRegisterInfo *RI = STI.getRegisterInfo(); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| Register FPReg = getFPReg(STI); |
| Register SPReg = getSPReg(STI); |
| |
| // All calls are tail calls in GHC calling conv, and functions have no |
| // prologue/epilogue. |
| if (MF.getFunction().getCallingConv() == CallingConv::GHC) |
| return; |
| |
| // Get the insert location for the epilogue. If there were no terminators in |
| // the block, get the last instruction. |
| MachineBasicBlock::iterator MBBI = MBB.end(); |
| DebugLoc DL; |
| if (!MBB.empty()) { |
| MBBI = MBB.getLastNonDebugInstr(); |
| if (MBBI != MBB.end()) |
| DL = MBBI->getDebugLoc(); |
| |
| MBBI = MBB.getFirstTerminator(); |
| |
| // If callee-saved registers are saved via libcall, place stack adjustment |
| // before this call. |
| while (MBBI != MBB.begin() && |
| std::prev(MBBI)->getFlag(MachineInstr::FrameDestroy)) |
| --MBBI; |
| } |
| |
| const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo()); |
| |
| // Skip to before the restores of callee-saved registers |
| // FIXME: assumes exactly one instruction is used to restore each |
| // callee-saved register. |
| auto LastFrameDestroy = MBBI; |
| if (!CSI.empty()) |
| LastFrameDestroy = std::prev(MBBI, CSI.size()); |
| |
| uint64_t StackSize = getStackSizeWithRVVPadding(MF); |
| uint64_t RealStackSize = StackSize + RVFI->getLibCallStackSize(); |
| uint64_t FPOffset = RealStackSize - RVFI->getVarArgsSaveSize(); |
| uint64_t RVVStackSize = RVFI->getRVVStackSize(); |
| |
| // Restore the stack pointer using the value of the frame pointer. Only |
| // necessary if the stack pointer was modified, meaning the stack size is |
| // unknown. |
| // |
| // In order to make sure the stack point is right through the EH region, |
| // we also need to restore stack pointer from the frame pointer if we |
| // don't preserve stack space within prologue/epilogue for outgoing variables, |
| // normally it's just checking the variable sized object is present or not |
| // is enough, but we also don't preserve that at prologue/epilogue when |
| // have vector objects in stack. |
| if (RI->hasStackRealignment(MF) || MFI.hasVarSizedObjects() || |
| !hasReservedCallFrame(MF)) { |
| assert(hasFP(MF) && "frame pointer should not have been eliminated"); |
| RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, FPReg, |
| StackOffset::getFixed(-FPOffset), |
| MachineInstr::FrameDestroy, getStackAlign()); |
| } else { |
| if (RVVStackSize) |
| adjustStackForRVV(MF, MBB, LastFrameDestroy, DL, RVVStackSize, |
| MachineInstr::FrameDestroy); |
| } |
| |
| uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
| if (FirstSPAdjustAmount) { |
| uint64_t SecondSPAdjustAmount = |
| getStackSizeWithRVVPadding(MF) - FirstSPAdjustAmount; |
| assert(SecondSPAdjustAmount > 0 && |
| "SecondSPAdjustAmount should be greater than zero"); |
| |
| RI->adjustReg(MBB, LastFrameDestroy, DL, SPReg, SPReg, |
| StackOffset::getFixed(SecondSPAdjustAmount), |
| MachineInstr::FrameDestroy, getStackAlign()); |
| } |
| |
| if (FirstSPAdjustAmount) |
| StackSize = FirstSPAdjustAmount; |
| |
| // Deallocate stack |
| RI->adjustReg(MBB, MBBI, DL, SPReg, SPReg, StackOffset::getFixed(StackSize), |
| MachineInstr::FrameDestroy, getStackAlign()); |
| |
| // Emit epilogue for shadow call stack. |
| emitSCSEpilogue(MF, MBB, MBBI, DL); |
| } |
| |
| StackOffset |
| RISCVFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, |
| Register &FrameReg) const { |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo(); |
| const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| |
| // Callee-saved registers should be referenced relative to the stack |
| // pointer (positive offset), otherwise use the frame pointer (negative |
| // offset). |
| const auto &CSI = getNonLibcallCSI(MF, MFI.getCalleeSavedInfo()); |
| int MinCSFI = 0; |
| int MaxCSFI = -1; |
| StackOffset Offset; |
| auto StackID = MFI.getStackID(FI); |
| |
| assert((StackID == TargetStackID::Default || |
| StackID == TargetStackID::ScalableVector) && |
| "Unexpected stack ID for the frame object."); |
| if (StackID == TargetStackID::Default) { |
| Offset = |
| StackOffset::getFixed(MFI.getObjectOffset(FI) - getOffsetOfLocalArea() + |
| MFI.getOffsetAdjustment()); |
| } else if (StackID == TargetStackID::ScalableVector) { |
| Offset = StackOffset::getScalable(MFI.getObjectOffset(FI)); |
| } |
| |
| uint64_t FirstSPAdjustAmount = getFirstSPAdjustAmount(MF); |
| |
| if (CSI.size()) { |
| MinCSFI = CSI[0].getFrameIdx(); |
| MaxCSFI = CSI[CSI.size() - 1].getFrameIdx(); |
| } |
| |
| if (FI >= MinCSFI && FI <= MaxCSFI) { |
| FrameReg = RISCV::X2; |
| |
| if (FirstSPAdjustAmount) |
| Offset += StackOffset::getFixed(FirstSPAdjustAmount); |
| else |
| Offset += StackOffset::getFixed(getStackSizeWithRVVPadding(MF)); |
| return Offset; |
| } |
| |
| if (RI->hasStackRealignment(MF) && !MFI.isFixedObjectIndex(FI)) { |
| // If the stack was realigned, the frame pointer is set in order to allow |
| // SP to be restored, so we need another base register to record the stack |
| // after realignment. |
| // |--------------------------| -- <-- FP |
| // | callee-allocated save | | <----| |
| // | area for register varargs| | | |
| // |--------------------------| | | |
| // | callee-saved registers | | | |
| // |--------------------------| -- | |
| // | realignment (the size of | | | |
| // | this area is not counted | | | |
| // | in MFI.getStackSize()) | | | |
| // |--------------------------| -- |-- MFI.getStackSize() |
| // | RVV alignment padding | | | |
| // | (not counted in | | | |
| // | MFI.getStackSize() but | | | |
| // | counted in | | | |
| // | RVFI.getRVVStackSize()) | | | |
| // |--------------------------| -- | |
| // | RVV objects | | | |
| // | (not counted in | | | |
| // | MFI.getStackSize()) | | | |
| // |--------------------------| -- | |
| // | padding before RVV | | | |
| // | (not counted in | | | |
| // | MFI.getStackSize() or in | | | |
| // | RVFI.getRVVStackSize()) | | | |
| // |--------------------------| -- | |
| // | scalar local variables | | <----' |
| // |--------------------------| -- <-- BP (if var sized objects present) |
| // | VarSize objects | | |
| // |--------------------------| -- <-- SP |
| if (hasBP(MF)) { |
| FrameReg = RISCVABI::getBPReg(); |
| } else { |
| // VarSize objects must be empty in this case! |
| assert(!MFI.hasVarSizedObjects()); |
| FrameReg = RISCV::X2; |
| } |
| } else { |
| FrameReg = RI->getFrameRegister(MF); |
| } |
| |
| if (FrameReg == getFPReg(STI)) { |
| Offset += StackOffset::getFixed(RVFI->getVarArgsSaveSize()); |
| if (FI >= 0) |
| Offset -= StackOffset::getFixed(RVFI->getLibCallStackSize()); |
| // When using FP to access scalable vector objects, we need to minus |
| // the frame size. |
| // |
| // |--------------------------| -- <-- FP |
| // | callee-allocated save | | |
| // | area for register varargs| | |
| // |--------------------------| | |
| // | callee-saved registers | | |
| // |--------------------------| | MFI.getStackSize() |
| // | scalar local variables | | |
| // |--------------------------| -- (Offset of RVV objects is from here.) |
| // | RVV objects | |
| // |--------------------------| |
| // | VarSize objects | |
| // |--------------------------| <-- SP |
| if (MFI.getStackID(FI) == TargetStackID::ScalableVector) { |
| assert(!RI->hasStackRealignment(MF) && |
| "Can't index across variable sized realign"); |
| // We don't expect any extra RVV alignment padding, as the stack size |
| // and RVV object sections should be correct aligned in their own |
| // right. |
| assert(MFI.getStackSize() == getStackSizeWithRVVPadding(MF) && |
| "Inconsistent stack layout"); |
| Offset -= StackOffset::getFixed(MFI.getStackSize()); |
| } |
| return Offset; |
| } |
| |
| // This case handles indexing off both SP and BP. |
| // If indexing off SP, there must not be any var sized objects |
| assert(FrameReg == RISCVABI::getBPReg() || !MFI.hasVarSizedObjects()); |
| |
| // When using SP to access frame objects, we need to add RVV stack size. |
| // |
| // |--------------------------| -- <-- FP |
| // | callee-allocated save | | <----| |
| // | area for register varargs| | | |
| // |--------------------------| | | |
| // | callee-saved registers | | | |
| // |--------------------------| -- | |
| // | RVV alignment padding | | | |
| // | (not counted in | | | |
| // | MFI.getStackSize() but | | | |
| // | counted in | | | |
| // | RVFI.getRVVStackSize()) | | | |
| // |--------------------------| -- | |
| // | RVV objects | | |-- MFI.getStackSize() |
| // | (not counted in | | | |
| // | MFI.getStackSize()) | | | |
| // |--------------------------| -- | |
| // | padding before RVV | | | |
| // | (not counted in | | | |
| // | MFI.getStackSize()) | | | |
| // |--------------------------| -- | |
| // | scalar local variables | | <----' |
| // |--------------------------| -- <-- BP (if var sized objects present) |
| // | VarSize objects | | |
| // |--------------------------| -- <-- SP |
| // |
| // The total amount of padding surrounding RVV objects is described by |
| // RVV->getRVVPadding() and it can be zero. It allows us to align the RVV |
| // objects to the required alignment. |
| if (MFI.getStackID(FI) == TargetStackID::Default) { |
| if (MFI.isFixedObjectIndex(FI)) { |
| assert(!RI->hasStackRealignment(MF) && |
| "Can't index across variable sized realign"); |
| Offset += StackOffset::get(getStackSizeWithRVVPadding(MF) + |
| RVFI->getLibCallStackSize(), |
| RVFI->getRVVStackSize()); |
| } else { |
| Offset += StackOffset::getFixed(MFI.getStackSize()); |
| } |
| } else if (MFI.getStackID(FI) == TargetStackID::ScalableVector) { |
| // Ensure the base of the RVV stack is correctly aligned: add on the |
| // alignment padding. |
| int ScalarLocalVarSize = |
| MFI.getStackSize() - RVFI->getCalleeSavedStackSize() - |
| RVFI->getVarArgsSaveSize() + RVFI->getRVVPadding(); |
| Offset += StackOffset::get(ScalarLocalVarSize, RVFI->getRVVStackSize()); |
| } |
| return Offset; |
| } |
| |
| void RISCVFrameLowering::determineCalleeSaves(MachineFunction &MF, |
| BitVector &SavedRegs, |
| RegScavenger *RS) const { |
| TargetFrameLowering::determineCalleeSaves(MF, SavedRegs, RS); |
| // Unconditionally spill RA and FP only if the function uses a frame |
| // pointer. |
| if (hasFP(MF)) { |
| SavedRegs.set(RISCV::X1); |
| SavedRegs.set(RISCV::X8); |
| } |
| // Mark BP as used if function has dedicated base pointer. |
| if (hasBP(MF)) |
| SavedRegs.set(RISCVABI::getBPReg()); |
| |
| // If interrupt is enabled and there are calls in the handler, |
| // unconditionally save all Caller-saved registers and |
| // all FP registers, regardless whether they are used. |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| |
| if (MF.getFunction().hasFnAttribute("interrupt") && MFI.hasCalls()) { |
| |
| static const MCPhysReg CSRegs[] = { RISCV::X1, /* ra */ |
| RISCV::X5, RISCV::X6, RISCV::X7, /* t0-t2 */ |
| RISCV::X10, RISCV::X11, /* a0-a1, a2-a7 */ |
| RISCV::X12, RISCV::X13, RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17, |
| RISCV::X28, RISCV::X29, RISCV::X30, RISCV::X31, 0 /* t3-t6 */ |
| }; |
| |
| for (unsigned i = 0; CSRegs[i]; ++i) |
| SavedRegs.set(CSRegs[i]); |
| |
| if (MF.getSubtarget<RISCVSubtarget>().hasStdExtF()) { |
| |
| // If interrupt is enabled, this list contains all FP registers. |
| const MCPhysReg * Regs = MF.getRegInfo().getCalleeSavedRegs(); |
| |
| for (unsigned i = 0; Regs[i]; ++i) |
| if (RISCV::FPR16RegClass.contains(Regs[i]) || |
| RISCV::FPR32RegClass.contains(Regs[i]) || |
| RISCV::FPR64RegClass.contains(Regs[i])) |
| SavedRegs.set(Regs[i]); |
| } |
| } |
| } |
| |
| std::pair<int64_t, Align> |
| RISCVFrameLowering::assignRVVStackObjectOffsets(MachineFunction &MF) const { |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| // Create a buffer of RVV objects to allocate. |
| SmallVector<int, 8> ObjectsToAllocate; |
| for (int I = 0, E = MFI.getObjectIndexEnd(); I != E; ++I) { |
| unsigned StackID = MFI.getStackID(I); |
| if (StackID != TargetStackID::ScalableVector) |
| continue; |
| if (MFI.isDeadObjectIndex(I)) |
| continue; |
| |
| ObjectsToAllocate.push_back(I); |
| } |
| |
| // The minimum alignment is 16 bytes. |
| Align RVVStackAlign(16); |
| const auto &ST = MF.getSubtarget<RISCVSubtarget>(); |
| |
| if (!ST.hasVInstructions()) { |
| assert(ObjectsToAllocate.empty() && |
| "Can't allocate scalable-vector objects without V instructions"); |
| return std::make_pair(0, RVVStackAlign); |
| } |
| |
| // Allocate all RVV locals and spills |
| int64_t Offset = 0; |
| for (int FI : ObjectsToAllocate) { |
| // ObjectSize in bytes. |
| int64_t ObjectSize = MFI.getObjectSize(FI); |
| auto ObjectAlign = std::max(Align(8), MFI.getObjectAlign(FI)); |
| // If the data type is the fractional vector type, reserve one vector |
| // register for it. |
| if (ObjectSize < 8) |
| ObjectSize = 8; |
| Offset = alignTo(Offset + ObjectSize, ObjectAlign); |
| MFI.setObjectOffset(FI, -Offset); |
| // Update the maximum alignment of the RVV stack section |
| RVVStackAlign = std::max(RVVStackAlign, ObjectAlign); |
| } |
| |
| // Ensure the alignment of the RVV stack. Since we want the most-aligned |
| // object right at the bottom (i.e., any padding at the top of the frame), |
| // readjust all RVV objects down by the alignment padding. |
| uint64_t StackSize = Offset; |
| if (auto AlignmentPadding = offsetToAlignment(StackSize, RVVStackAlign)) { |
| StackSize += AlignmentPadding; |
| for (int FI : ObjectsToAllocate) |
| MFI.setObjectOffset(FI, MFI.getObjectOffset(FI) - AlignmentPadding); |
| } |
| |
| return std::make_pair(StackSize, RVVStackAlign); |
| } |
| |
| static unsigned getScavSlotsNumForRVV(MachineFunction &MF) { |
| // For RVV spill, scalable stack offsets computing requires up to two scratch |
| // registers |
| static constexpr unsigned ScavSlotsNumRVVSpillScalableObject = 2; |
| |
| // For RVV spill, non-scalable stack offsets computing requires up to one |
| // scratch register. |
| static constexpr unsigned ScavSlotsNumRVVSpillNonScalableObject = 1; |
| |
| // ADDI instruction's destination register can be used for computing |
| // offsets. So Scalable stack offsets require up to one scratch register. |
| static constexpr unsigned ScavSlotsADDIScalableObject = 1; |
| |
| static constexpr unsigned MaxScavSlotsNumKnown = |
| std::max({ScavSlotsADDIScalableObject, ScavSlotsNumRVVSpillScalableObject, |
| ScavSlotsNumRVVSpillNonScalableObject}); |
| |
| unsigned MaxScavSlotsNum = 0; |
| if (!MF.getSubtarget<RISCVSubtarget>().hasVInstructions()) |
| return false; |
| for (const MachineBasicBlock &MBB : MF) |
| for (const MachineInstr &MI : MBB) { |
| bool IsRVVSpill = RISCV::isRVVSpill(MI); |
| for (auto &MO : MI.operands()) { |
| if (!MO.isFI()) |
| continue; |
| bool IsScalableVectorID = MF.getFrameInfo().getStackID(MO.getIndex()) == |
| TargetStackID::ScalableVector; |
| if (IsRVVSpill) { |
| MaxScavSlotsNum = std::max( |
| MaxScavSlotsNum, IsScalableVectorID |
| ? ScavSlotsNumRVVSpillScalableObject |
| : ScavSlotsNumRVVSpillNonScalableObject); |
| } else if (MI.getOpcode() == RISCV::ADDI && IsScalableVectorID) { |
| MaxScavSlotsNum = |
| std::max(MaxScavSlotsNum, ScavSlotsADDIScalableObject); |
| } |
| } |
| if (MaxScavSlotsNum == MaxScavSlotsNumKnown) |
| return MaxScavSlotsNumKnown; |
| } |
| return MaxScavSlotsNum; |
| } |
| |
| static bool hasRVVFrameObject(const MachineFunction &MF) { |
| // Originally, the function will scan all the stack objects to check whether |
| // if there is any scalable vector object on the stack or not. However, it |
| // causes errors in the register allocator. In issue 53016, it returns false |
| // before RA because there is no RVV stack objects. After RA, it returns true |
| // because there are spilling slots for RVV values during RA. It will not |
| // reserve BP during register allocation and generate BP access in the PEI |
| // pass due to the inconsistent behavior of the function. |
| // |
| // The function is changed to use hasVInstructions() as the return value. It |
| // is not precise, but it can make the register allocation correct. |
| // |
| // FIXME: Find a better way to make the decision or revisit the solution in |
| // D103622. |
| // |
| // Refer to https://github.com/llvm/llvm-project/issues/53016. |
| return MF.getSubtarget<RISCVSubtarget>().hasVInstructions(); |
| } |
| |
| static unsigned estimateFunctionSizeInBytes(const MachineFunction &MF, |
| const RISCVInstrInfo &TII) { |
| unsigned FnSize = 0; |
| for (auto &MBB : MF) { |
| for (auto &MI : MBB) { |
| // Far branches over 20-bit offset will be relaxed in branch relaxation |
| // pass. In the worst case, conditional branches will be relaxed into |
| // the following instruction sequence. Unconditional branches are |
| // relaxed in the same way, with the exception that there is no first |
| // branch instruction. |
| // |
| // foo |
| // bne t5, t6, .rev_cond # `TII->getInstSizeInBytes(MI)` bytes |
| // sd s11, 0(sp) # 4 bytes, or 2 bytes in RVC |
| // jump .restore, s11 # 8 bytes |
| // .rev_cond |
| // bar |
| // j .dest_bb # 4 bytes, or 2 bytes in RVC |
| // .restore: |
| // ld s11, 0(sp) # 4 bytes, or 2 bytes in RVC |
| // .dest: |
| // baz |
| if (MI.isConditionalBranch()) |
| FnSize += TII.getInstSizeInBytes(MI); |
| if (MI.isConditionalBranch() || MI.isUnconditionalBranch()) { |
| if (MF.getSubtarget<RISCVSubtarget>().hasStdExtC()) |
| FnSize += 2 + 8 + 2 + 2; |
| else |
| FnSize += 4 + 8 + 4 + 4; |
| continue; |
| } |
| |
| FnSize += TII.getInstSizeInBytes(MI); |
| } |
| } |
| return FnSize; |
| } |
| |
| void RISCVFrameLowering::processFunctionBeforeFrameFinalized( |
| MachineFunction &MF, RegScavenger *RS) const { |
| const RISCVRegisterInfo *RegInfo = |
| MF.getSubtarget<RISCVSubtarget>().getRegisterInfo(); |
| const RISCVInstrInfo *TII = MF.getSubtarget<RISCVSubtarget>().getInstrInfo(); |
| MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const TargetRegisterClass *RC = &RISCV::GPRRegClass; |
| auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| |
| int64_t RVVStackSize; |
| Align RVVStackAlign; |
| std::tie(RVVStackSize, RVVStackAlign) = assignRVVStackObjectOffsets(MF); |
| |
| RVFI->setRVVStackSize(RVVStackSize); |
| RVFI->setRVVStackAlign(RVVStackAlign); |
| |
| if (hasRVVFrameObject(MF)) { |
| // Ensure the entire stack is aligned to at least the RVV requirement: some |
| // scalable-vector object alignments are not considered by the |
| // target-independent code. |
| MFI.ensureMaxAlignment(RVVStackAlign); |
| } |
| |
| unsigned ScavSlotsNum = 0; |
| |
| // estimateStackSize has been observed to under-estimate the final stack |
| // size, so give ourselves wiggle-room by checking for stack size |
| // representable an 11-bit signed field rather than 12-bits. |
| if (!isInt<11>(MFI.estimateStackSize(MF))) |
| ScavSlotsNum = 1; |
| |
| // Far branches over 20-bit offset require a spill slot for scratch register. |
| bool IsLargeFunction = !isInt<20>(estimateFunctionSizeInBytes(MF, *TII)); |
| if (IsLargeFunction) |
| ScavSlotsNum = std::max(ScavSlotsNum, 1u); |
| |
| // RVV loads & stores have no capacity to hold the immediate address offsets |
| // so we must always reserve an emergency spill slot if the MachineFunction |
| // contains any RVV spills. |
| ScavSlotsNum = std::max(ScavSlotsNum, getScavSlotsNumForRVV(MF)); |
| |
| for (unsigned I = 0; I < ScavSlotsNum; I++) { |
| int FI = MFI.CreateStackObject(RegInfo->getSpillSize(*RC), |
| RegInfo->getSpillAlign(*RC), false); |
| RS->addScavengingFrameIndex(FI); |
| |
| if (IsLargeFunction && RVFI->getBranchRelaxationScratchFrameIndex() == -1) |
| RVFI->setBranchRelaxationScratchFrameIndex(FI); |
| } |
| |
| if (MFI.getCalleeSavedInfo().empty() || RVFI->useSaveRestoreLibCalls(MF)) { |
| RVFI->setCalleeSavedStackSize(0); |
| return; |
| } |
| |
| unsigned Size = 0; |
| for (const auto &Info : MFI.getCalleeSavedInfo()) { |
| int FrameIdx = Info.getFrameIdx(); |
| if (MFI.getStackID(FrameIdx) != TargetStackID::Default) |
| continue; |
| |
| Size += MFI.getObjectSize(FrameIdx); |
| } |
| RVFI->setCalleeSavedStackSize(Size); |
| } |
| |
| // Not preserve stack space within prologue for outgoing variables when the |
| // function contains variable size objects or there are vector objects accessed |
| // by the frame pointer. |
| // Let eliminateCallFramePseudoInstr preserve stack space for it. |
| bool RISCVFrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { |
| return !MF.getFrameInfo().hasVarSizedObjects() && |
| !(hasFP(MF) && hasRVVFrameObject(MF)); |
| } |
| |
| // Eliminate ADJCALLSTACKDOWN, ADJCALLSTACKUP pseudo instructions. |
| MachineBasicBlock::iterator RISCVFrameLowering::eliminateCallFramePseudoInstr( |
| MachineFunction &MF, MachineBasicBlock &MBB, |
| MachineBasicBlock::iterator MI) const { |
| Register SPReg = RISCV::X2; |
| DebugLoc DL = MI->getDebugLoc(); |
| |
| if (!hasReservedCallFrame(MF)) { |
| // If space has not been reserved for a call frame, ADJCALLSTACKDOWN and |
| // ADJCALLSTACKUP must be converted to instructions manipulating the stack |
| // pointer. This is necessary when there is a variable length stack |
| // allocation (e.g. alloca), which means it's not possible to allocate |
| // space for outgoing arguments from within the function prologue. |
| int64_t Amount = MI->getOperand(0).getImm(); |
| |
| if (Amount != 0) { |
| // Ensure the stack remains aligned after adjustment. |
| Amount = alignSPAdjust(Amount); |
| |
| if (MI->getOpcode() == RISCV::ADJCALLSTACKDOWN) |
| Amount = -Amount; |
| |
| const RISCVRegisterInfo &RI = *STI.getRegisterInfo(); |
| RI.adjustReg(MBB, MI, DL, SPReg, SPReg, StackOffset::getFixed(Amount), |
| MachineInstr::NoFlags, getStackAlign()); |
| } |
| } |
| |
| return MBB.erase(MI); |
| } |
| |
| // We would like to split the SP adjustment to reduce prologue/epilogue |
| // as following instructions. In this way, the offset of the callee saved |
| // register could fit in a single store. |
| // add sp,sp,-2032 |
| // sw ra,2028(sp) |
| // sw s0,2024(sp) |
| // sw s1,2020(sp) |
| // sw s3,2012(sp) |
| // sw s4,2008(sp) |
| // add sp,sp,-64 |
| uint64_t |
| RISCVFrameLowering::getFirstSPAdjustAmount(const MachineFunction &MF) const { |
| const auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>(); |
| const MachineFrameInfo &MFI = MF.getFrameInfo(); |
| const std::vector<CalleeSavedInfo> &CSI = MFI.getCalleeSavedInfo(); |
| uint64_t StackSize = getStackSizeWithRVVPadding(MF); |
| |
| // Disable SplitSPAdjust if save-restore libcall is used. The callee-saved |
| // registers will be pushed by the save-restore libcalls, so we don't have to |
| // split the SP adjustment in this case. |
| if (RVFI->getLibCallStackSize()) |
| return 0; |
| |
| // Return the FirstSPAdjustAmount if the StackSize can not fit in a signed |
| // 12-bit and there exists a callee-saved register needing to be pushed. |
| if (!isInt<12>(StackSize) && (CSI.size() > 0)) { |
| // FirstSPAdjustAmount is chosen as (2048 - StackAlign) because 2048 will |
| // cause sp = sp + 2048 in the epilogue to be split into multiple |
| // instructions. Offsets smaller than 2048 can fit in a single load/store |
| // instruction, and we have to stick with the stack alignment. 2048 has |
| // 16-byte alignment. The stack alignment for RV32 and RV64 is 16 and for |
| // RV32E it is 4. So (2048 - StackAlign) will satisfy the stack alignment. |
| return 2048 - getStackAlign().value(); |
| } |
| return 0; |
| } |
| |
| bool RISCVFrameLowering::spillCalleeSavedRegisters( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| ArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
| if (CSI.empty()) |
| return true; |
| |
| MachineFunction *MF = MBB.getParent(); |
| const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); |
| DebugLoc DL; |
| if (MI != MBB.end() && !MI->isDebugInstr()) |
| DL = MI->getDebugLoc(); |
| |
| const char *SpillLibCall = getSpillLibCallName(*MF, CSI); |
| if (SpillLibCall) { |
| // Add spill libcall via non-callee-saved register t0. |
| BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoCALLReg), RISCV::X5) |
| .addExternalSymbol(SpillLibCall, RISCVII::MO_CALL) |
| .setMIFlag(MachineInstr::FrameSetup); |
| |
| // Add registers spilled in libcall as liveins. |
| for (auto &CS : CSI) |
| MBB.addLiveIn(CS.getReg()); |
| } |
| |
| // Manually spill values not spilled by libcall. |
| const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI); |
| for (auto &CS : NonLibcallCSI) { |
| // Insert the spill to the stack frame. |
| Register Reg = CS.getReg(); |
| const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
| TII.storeRegToStackSlot(MBB, MI, Reg, !MBB.isLiveIn(Reg), CS.getFrameIdx(), |
| RC, TRI, Register()); |
| } |
| |
| return true; |
| } |
| |
| bool RISCVFrameLowering::restoreCalleeSavedRegisters( |
| MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, |
| MutableArrayRef<CalleeSavedInfo> CSI, const TargetRegisterInfo *TRI) const { |
| if (CSI.empty()) |
| return true; |
| |
| MachineFunction *MF = MBB.getParent(); |
| const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo(); |
| DebugLoc DL; |
| if (MI != MBB.end() && !MI->isDebugInstr()) |
| DL = MI->getDebugLoc(); |
| |
| // Manually restore values not restored by libcall. |
| // Keep the same order as in the prologue. There is no need to reverse the |
| // order in the epilogue. In addition, the return address will be restored |
| // first in the epilogue. It increases the opportunity to avoid the |
| // load-to-use data hazard between loading RA and return by RA. |
| // loadRegFromStackSlot can insert multiple instructions. |
| const auto &NonLibcallCSI = getNonLibcallCSI(*MF, CSI); |
| for (auto &CS : NonLibcallCSI) { |
| Register Reg = CS.getReg(); |
| const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); |
| TII.loadRegFromStackSlot(MBB, MI, Reg, CS.getFrameIdx(), RC, TRI, |
| Register()); |
| assert(MI != MBB.begin() && "loadRegFromStackSlot didn't insert any code!"); |
| } |
| |
| const char *RestoreLibCall = getRestoreLibCallName(*MF, CSI); |
| if (RestoreLibCall) { |
| // Add restore libcall via tail call. |
| MachineBasicBlock::iterator NewMI = |
| BuildMI(MBB, MI, DL, TII.get(RISCV::PseudoTAIL)) |
| .addExternalSymbol(RestoreLibCall, RISCVII::MO_CALL) |
| .setMIFlag(MachineInstr::FrameDestroy); |
| |
| // Remove trailing returns, since the terminator is now a tail call to the |
| // restore function. |
| if (MI != MBB.end() && MI->getOpcode() == RISCV::PseudoRET) { |
| NewMI->copyImplicitOps(*MF, *MI); |
| MI->eraseFromParent(); |
| } |
| } |
| |
| return true; |
| } |
| |
| bool RISCVFrameLowering::enableShrinkWrapping(const MachineFunction &MF) const { |
| // Keep the conventional code flow when not optimizing. |
| if (MF.getFunction().hasOptNone()) |
| return false; |
| |
| return true; |
| } |
| |
| bool RISCVFrameLowering::canUseAsPrologue(const MachineBasicBlock &MBB) const { |
| MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB); |
| const MachineFunction *MF = MBB.getParent(); |
| const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
| |
| if (!RVFI->useSaveRestoreLibCalls(*MF)) |
| return true; |
| |
| // Inserting a call to a __riscv_save libcall requires the use of the register |
| // t0 (X5) to hold the return address. Therefore if this register is already |
| // used we can't insert the call. |
| |
| RegScavenger RS; |
| RS.enterBasicBlock(*TmpMBB); |
| return !RS.isRegUsed(RISCV::X5); |
| } |
| |
| bool RISCVFrameLowering::canUseAsEpilogue(const MachineBasicBlock &MBB) const { |
| const MachineFunction *MF = MBB.getParent(); |
| MachineBasicBlock *TmpMBB = const_cast<MachineBasicBlock *>(&MBB); |
| const auto *RVFI = MF->getInfo<RISCVMachineFunctionInfo>(); |
| |
| if (!RVFI->useSaveRestoreLibCalls(*MF)) |
| return true; |
| |
| // Using the __riscv_restore libcalls to restore CSRs requires a tail call. |
| // This means if we still need to continue executing code within this function |
| // the restore cannot take place in this basic block. |
| |
| if (MBB.succ_size() > 1) |
| return false; |
| |
| MachineBasicBlock *SuccMBB = |
| MBB.succ_empty() ? TmpMBB->getFallThrough() : *MBB.succ_begin(); |
| |
| // Doing a tail call should be safe if there are no successors, because either |
| // we have a returning block or the end of the block is unreachable, so the |
| // restore will be eliminated regardless. |
| if (!SuccMBB) |
| return true; |
| |
| // The successor can only contain a return, since we would effectively be |
| // replacing the successor with our own tail return at the end of our block. |
| return SuccMBB->isReturnBlock() && SuccMBB->size() == 1; |
| } |
| |
| bool RISCVFrameLowering::isSupportedStackID(TargetStackID::Value ID) const { |
| switch (ID) { |
| case TargetStackID::Default: |
| case TargetStackID::ScalableVector: |
| return true; |
| case TargetStackID::NoAlloc: |
| case TargetStackID::SGPRSpill: |
| case TargetStackID::WasmLocal: |
| return false; |
| } |
| llvm_unreachable("Invalid TargetStackID::Value"); |
| } |
| |
| TargetStackID::Value RISCVFrameLowering::getStackIDForScalableVectors() const { |
| return TargetStackID::ScalableVector; |
| } |
