third_party/llvm-16.0/configs/common/lib/Target/ARM/ARMGenFastISel.inc

/*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
|*                                                                            *|
|* "Fast" Instruction Selector for the ARM target                             *|
|*                                                                            *|
|* Automatically generated file, do not edit!                                 *|
|*                                                                            *|
\*===----------------------------------------------------------------------===*/


// FastEmit Immediate Predicate functions.
static bool Predicate_mod_imm(int64_t Imm) {

    return ARM_AM::getSOImmVal(Imm) != -1;
  
}
static bool Predicate_imm0_65535(int64_t Imm) {

  return Imm >= 0 && Imm < 65536;

}
static bool Predicate_imm0_7(int64_t Imm) {

  return Imm >= 0 && Imm < 8;

}
static bool Predicate_imm8_255(int64_t Imm) {

  return Imm >= 8 && Imm < 256;

}
static bool Predicate_imm0_255(int64_t Imm) {
 return Imm >= 0 && Imm < 256; 
}
static bool Predicate_t2_so_imm(int64_t Imm) {

    return ARM_AM::getT2SOImmVal(Imm) != -1;
  
}
static bool Predicate_imm0_4095(int64_t Imm) {

  return Imm >= 0 && Imm < 4096;

}
static bool Predicate_imm1_31(int64_t Imm) {
 return Imm > 0 && Imm < 32; 
}
static bool Predicate_shr_imm8(int64_t Imm) {
 return Imm > 0 && Imm <= 8; 
}
static bool Predicate_shr_imm16(int64_t Imm) {
 return Imm > 0 && Imm <= 16; 
}
static bool Predicate_shr_imm32(int64_t Imm) {
 return Imm > 0 && Imm <= 32; 
}
static bool Predicate_VectorIndex32(int64_t Imm) {

  return ((uint64_t)Imm) < 2;

}
static bool Predicate_imm0_31(int64_t Imm) {

  return Imm >= 0 && Imm < 32;

}
static bool Predicate_t2_so_imm_neg(int64_t Imm) {

  return Imm && ARM_AM::getT2SOImmVal(-(uint32_t)Imm) != -1;

}
static bool Predicate_imm0_15(int64_t Imm) {

  return Imm >= 0 && Imm < 16;

}


// FastEmit functions for ARMISD::CALL.

unsigned fastEmit_ARMISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
    return fastEmitInst_r(ARM::BLX_noip, &ARM::GPRnoipRegClass, Op0);
  }
  if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
    return fastEmitInst_r(ARM::BLX, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_CALL_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::CALL_NOLINK.

unsigned fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
    return fastEmitInst_r(ARM::tBX_CALL, &ARM::tGPRRegClass, Op0);
  }
  if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
    return fastEmitInst_r(ARM::BMOVPCRX_CALL, &ARM::tGPRRegClass, Op0);
  }
  if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::BX_CALL, &ARM::tGPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CALL_NOLINK_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_CALL_NOLINK_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::CALL_PRED.

unsigned fastEmit_ARMISD_CALL_PRED_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
    return fastEmitInst_r(ARM::BLX_pred_noip, &ARM::GPRnoipRegClass, Op0);
  }
  if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb()) && ( !MF->getSubtarget<ARMSubtarget>().hardenSlsBlr() )) {
    return fastEmitInst_r(ARM::BLX_pred, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CALL_PRED_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_CALL_PRED_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::CMPFPEw0.

unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VCMPEZH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VCMPEZS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VCMPEZD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPEw0_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ARMISD_CMPFPEw0_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ARMISD_CMPFPEw0_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ARMISD_CMPFPEw0_MVT_f64_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::CMPFPw0.

unsigned fastEmit_ARMISD_CMPFPw0_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VCMPZH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPw0_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VCMPZS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPw0_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VCMPZD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_CMPFPw0_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ARMISD_CMPFPw0_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ARMISD_CMPFPw0_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ARMISD_CMPFPw0_MVT_f64_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::RRX.

unsigned fastEmit_ARMISD_RRX_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2RRX, &ARM::rGPRRegClass, Op0);
  }
  if ((!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::RRX, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_RRX_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_RRX_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::SRA_FLAG.

unsigned fastEmit_ARMISD_SRA_FLAG_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2MOVsra_flag, &ARM::rGPRRegClass, Op0);
  }
  if ((!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::MOVsra_flag, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_SRA_FLAG_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_SRA_FLAG_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::SRL_FLAG.

unsigned fastEmit_ARMISD_SRL_FLAG_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2MOVsrl_flag, &ARM::rGPRRegClass, Op0);
  }
  if ((!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::MOVsrl_flag, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_SRL_FLAG_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_SRL_FLAG_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VADDVs.

unsigned fastEmit_ARMISD_VADDVs_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVs8no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVs_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVs16no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVs_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVs32no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVs_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ARMISD_VADDVs_MVT_v16i8_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ARMISD_VADDVs_MVT_v8i16_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ARMISD_VADDVs_MVT_v4i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VADDVu.

unsigned fastEmit_ARMISD_VADDVu_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVu8no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVu_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVu16no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVu_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VADDVu32no_acc, &ARM::tGPREvenRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VADDVu_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ARMISD_VADDVu_MVT_v16i8_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ARMISD_VADDVu_MVT_v8i16_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ARMISD_VADDVu_MVT_v4i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VDUP.

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP8d, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VDUP8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP8q, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP16d, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VDUP16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP16q, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(unsigned Op0) {
  if ((!Subtarget->hasSlowVDUP32()) && (Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP32d, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VDUP32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VDUP32q, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VDUP16, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VDUP32, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VDUP_MVT_i32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i8_r(Op0);
  case MVT::v16i8: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v16i8_r(Op0);
  case MVT::v4i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i16_r(Op0);
  case MVT::v8i16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8i16_r(Op0);
  case MVT::v2i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v2i32_r(Op0);
  case MVT::v4i32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4i32_r(Op0);
  case MVT::v8f16: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v8f16_r(Op0);
  case MVT::v4f32: return fastEmit_ARMISD_VDUP_MVT_i32_MVT_v4f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ARMISD_VDUP_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_VDUP_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VMOVSR.

unsigned fastEmit_ARMISD_VMOVSR_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((Subtarget->hasVFP2Base()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
    return fastEmitInst_r(ARM::VMOVSR, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VMOVSR_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_VMOVSR_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VMOVhr.

unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(unsigned Op0) {
  if ((Subtarget->hasFPRegs16())) {
    return fastEmitInst_r(ARM::VMOVHR, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(unsigned Op0) {
  if ((Subtarget->hasFPRegs16())) {
    return fastEmitInst_r(ARM::VMOVHR, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VMOVhr_MVT_i32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::bf16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_bf16_r(Op0);
  case MVT::f16: return fastEmit_ARMISD_VMOVhr_MVT_i32_MVT_f16_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ARMISD_VMOVhr_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_VMOVhr_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VMOVrh.

unsigned fastEmit_ARMISD_VMOVrh_MVT_bf16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasFPRegs16())) {
    return fastEmitInst_r(ARM::VMOVRH, &ARM::rGPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VMOVrh_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasFPRegs16())) {
    return fastEmitInst_r(ARM::VMOVRH, &ARM::rGPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VMOVrh_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::bf16: return fastEmit_ARMISD_VMOVrh_MVT_bf16_r(RetVT, Op0);
  case MVT::f16: return fastEmit_ARMISD_VMOVrh_MVT_f16_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VREV16.

unsigned fastEmit_ARMISD_VREV16_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV16_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV16_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ARMISD_VREV16_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ARMISD_VREV16_MVT_v16i8_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VREV32.

unsigned fastEmit_ARMISD_VREV32_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4bf16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8bf16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV32_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ARMISD_VREV32_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ARMISD_VREV32_MVT_v16i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ARMISD_VREV32_MVT_v4i16_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ARMISD_VREV32_MVT_v8i16_r(RetVT, Op0);
  case MVT::v4f16: return fastEmit_ARMISD_VREV32_MVT_v4f16_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ARMISD_VREV32_MVT_v8f16_r(RetVT, Op0);
  case MVT::v4bf16: return fastEmit_ARMISD_VREV32_MVT_v4bf16_r(RetVT, Op0);
  case MVT::v8bf16: return fastEmit_ARMISD_VREV32_MVT_v8bf16_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::VREV64.

unsigned fastEmit_ARMISD_VREV64_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2i32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4bf16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8bf16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2f32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_VREV64_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ARMISD_VREV64_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ARMISD_VREV64_MVT_v16i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ARMISD_VREV64_MVT_v4i16_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ARMISD_VREV64_MVT_v8i16_r(RetVT, Op0);
  case MVT::v2i32: return fastEmit_ARMISD_VREV64_MVT_v2i32_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ARMISD_VREV64_MVT_v4i32_r(RetVT, Op0);
  case MVT::v4f16: return fastEmit_ARMISD_VREV64_MVT_v4f16_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ARMISD_VREV64_MVT_v8f16_r(RetVT, Op0);
  case MVT::v4bf16: return fastEmit_ARMISD_VREV64_MVT_v4bf16_r(RetVT, Op0);
  case MVT::v8bf16: return fastEmit_ARMISD_VREV64_MVT_v8bf16_r(RetVT, Op0);
  case MVT::v2f32: return fastEmit_ARMISD_VREV64_MVT_v2f32_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ARMISD_VREV64_MVT_v4f32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::WIN__DBZCHK.

unsigned fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  return fastEmitInst_r(ARM::WIN__DBZCHK, &ARM::tGPRRegClass, Op0);
}

unsigned fastEmit_ARMISD_WIN__DBZCHK_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_WIN__DBZCHK_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ARMISD::tSECALL.

unsigned fastEmit_ARMISD_tSECALL_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->has8MSecExt()) && (Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::tBLXNS_CALL, &ARM::GPRnopcRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ARMISD_tSECALL_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ARMISD_tSECALL_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::ABS.

unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv8i8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VABSs8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv16i8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv4i16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VABSs16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv8i16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2i32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv2i32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VABSs32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSv4i32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0);
  case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::ANY_EXTEND.

unsigned fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VMOVLuv8i16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VMOVLuv4i32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2i64)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VMOVLuv2i64, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ISD_ANY_EXTEND_MVT_v8i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ISD_ANY_EXTEND_MVT_v4i16_r(RetVT, Op0);
  case MVT::v2i32: return fastEmit_ISD_ANY_EXTEND_MVT_v2i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::BITCAST.

unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((Subtarget->hasFPRegs()) && (Subtarget->preferVMOVSR() ||!Subtarget->useNEONForSinglePrecisionFP())) {
    return fastEmitInst_r(ARM::VMOVSR, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->hasFPRegs())) {
    return fastEmitInst_r(ARM::VMOVRS, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4bf16_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4bf16_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8bf16_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4bf16_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0);
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8bf16_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4bf16_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0);
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8bf16_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4bf16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_f64_r(Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v8i8_r(Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2i32_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v1i64_r(Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4bf16_MVT_v2f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV16q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v8bf16_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v16i8_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4i32_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2i64_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v4f32_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8bf16_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64d32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32d16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4bf16_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV32_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV32q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0);
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8bf16_r(Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(unsigned Op0) {
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0) {
  if ((Subtarget->hasMVEIntegerOps()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::MVE_VREV64_32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON()) && (MF->getDataLayout().isBigEndian())) {
    return fastEmitInst_r(ARM::VREV64q32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0) {
switch (RetVT.SimpleTy) {
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0);
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8bf16_r(Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0);
  default: return 0;
}
}

unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0);
  case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0);
  case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0);
  case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0);
  case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0);
  case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0);
  case MVT::v4bf16: return fastEmit_ISD_BITCAST_MVT_v4bf16_r(RetVT, Op0);
  case MVT::v8bf16: return fastEmit_ISD_BITCAST_MVT_v8bf16_r(RetVT, Op0);
  case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0);
  case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::BITREVERSE.

unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2RBIT, &ARM::rGPRRegClass, Op0);
  }
  if ((Subtarget->hasV6T2Ops()) && (!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::RBIT, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::BRIND.

unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::isVoid)
    return 0;
  if ((Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::tBRIND, &ARM::GPRRegClass, Op0);
  }
  if ((!Subtarget->isThumb()) && (!Subtarget->hasV4TOps())) {
    return fastEmitInst_r(ARM::MOVPCRX, &ARM::GPRRegClass, Op0);
  }
  if ((Subtarget->hasV4TOps()) && (!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::BX, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::BSWAP.

unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2REV, &ARM::rGPRRegClass, Op0);
  }
  if ((Subtarget->hasV6Ops()) && (Subtarget->isThumb()) && (Subtarget->isThumb1Only())) {
    return fastEmitInst_r(ARM::tREV, &ARM::tGPRRegClass, Op0);
  }
  if ((Subtarget->hasV6Ops()) && (!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::REV, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BSWAP_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV16_8, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BSWAP_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VREV32_8, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ISD_BSWAP_MVT_v8i16_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ISD_BSWAP_MVT_v4i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::CTLZ.

unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::i32)
    return 0;
  if ((Subtarget->isThumb2())) {
    return fastEmitInst_r(ARM::t2CLZ, &ARM::rGPRRegClass, Op0);
  }
  if ((Subtarget->hasV5TOps()) && (!Subtarget->isThumb())) {
    return fastEmitInst_r(ARM::CLZ, &ARM::GPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv8i8, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VCLZs8, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv16i8, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i16)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv4i16, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VCLZs16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv8i16, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2i32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv2i32, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4i32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VCLZs32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCLZv4i32, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0);
  case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0);
  case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0);
  case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0);
  case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0);
  case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::CTPOP.

unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCNTd, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v16i8)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VCNTq, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0);
  case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FABS.

unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f16)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VABSH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VABSS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VABSD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f16)
    return 0;
  if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABShd, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VABSf16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABShq, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2f32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSfd, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VABSf32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VABSfq, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0);
  case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0);
  case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FCEIL.

unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f16)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VRINTPH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTPS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTPD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEFloatOps())) {
    return fastEmitInst_r(ARM::MVE_VRINTf16P, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  if ((Subtarget->hasMVEFloatOps())) {
    return fastEmitInst_r(ARM::MVE_VRINTf32P, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FFLOOR.

unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f16)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VRINTMH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTMS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTMD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEFloatOps())) {
    return fastEmitInst_r(ARM::MVE_VRINTf16M, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  if ((Subtarget->hasMVEFloatOps())) {
    return fastEmitInst_r(ARM::MVE_VRINTf32M, &ARM::MQPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FNEARBYINT.

unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f16)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VRINTRH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTRS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasFPARMv8Base())) {
    return fastEmitInst_r(ARM::VRINTRD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FNEG.

unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f16)
    return 0;
  if ((Subtarget->hasFullFP16())) {
    return fastEmitInst_r(ARM::VNEGH, &ARM::HPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f32)
    return 0;
  if ((!Subtarget->useNEONForSinglePrecisionFP()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VNEGS, &ARM::SPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VNEGD, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f16)
    return 0;
  if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VNEGhd, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v8f16)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VNEGf16, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VNEGhq, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v2f32)
    return 0;
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VNEGfd, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  if ((Subtarget->hasMVEIntegerOps())) {
    return fastEmitInst_r(ARM::MVE_VNEGf32, &ARM::MQPRRegClass, Op0);
  }
  if ((Subtarget->hasNEON())) {
    return fastEmitInst_r(ARM::VNEGf32q, &ARM::QPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0);
  case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0);
  case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0);
  case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0);
  case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0);
  case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0);
  case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0);
  default: return 0;
  }
}

// FastEmit functions for ISD::FP_EXTEND.

unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::f64)
    return 0;
  if ((Subtarget->hasFP64()) && (Subtarget->hasVFP2Base())) {
    return fastEmitInst_r(ARM::VCVTDS, &ARM::DPRRegClass, Op0);
  }
  return 0;
}

unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0) {
  if (RetVT.SimpleTy != MVT::v4f32)
    return 0;
  return fastEmitInst_r(ARM::VCVTh2f, &ARM::QPRRegClass, Op0);
}

unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0) {
  switch (VT.SimpleTy) {
  case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0);