| //=-- SMEInstrFormats.td - AArch64 SME Instruction classes -*- tablegen -*--=// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // AArch64 Scalable Matrix Extension (SME) Instruction Class Definitions. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| def imm_to_tile8 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAB0>", []>; |
| def imm_to_tile16 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAH0>", []>; |
| def imm_to_tile32 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAS0>", []>; |
| def imm_to_tile64 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAD0>", []>; |
| def imm_to_tile128 : ComplexPattern<i32, 1, "ImmToTile<AArch64::ZAQ0>", []>; |
| |
| def tileslice8 : ComplexPattern<i32 , 2, "SelectSMETileSlice<15, 1>", []>; |
| def tileslice16 : ComplexPattern<i32 , 2, "SelectSMETileSlice<7, 1>", []>; |
| def tileslice32 : ComplexPattern<i32 , 2, "SelectSMETileSlice<3, 1>", []>; |
| def tileslice64 : ComplexPattern<i32 , 2, "SelectSMETileSlice<1, 1>", []>; |
| def tileslice128 : ComplexPattern<i32 , 2, "SelectSMETileSlice<0, 1>", []>; // nop |
| |
| def tileslicerange3s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<14, 2>", []>; |
| def tileslicerange2s2 : ComplexPattern<i32, 2, "SelectSMETileSlice<6, 2>", []>; |
| |
| def am_sme_indexed_b4 :ComplexPattern<iPTR, 2, "SelectAddrModeIndexedSVE<0,15>", [], [SDNPWantRoot]>; |
| |
| //===----------------------------------------------------------------------===// |
| // SME Pseudo Classes |
| //===----------------------------------------------------------------------===// |
| |
| def getSMEPseudoMap : InstrMapping { |
| let FilterClass = "SMEPseudo2Instr"; |
| let RowFields = ["PseudoName"]; |
| let ColFields = ["IsInstr"]; |
| let KeyCol = ["0"]; |
| let ValueCols = [["1"]]; |
| } |
| |
| class SMEPseudo2Instr<string name, bit instr> { |
| string PseudoName = name; |
| bit IsInstr = instr; |
| } |
| |
| class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> |
| : Pseudo<(outs), (ins i32imm:$tile, PPR3bAny:$pn, PPR3bAny:$pm, |
| zpr_ty:$zn, zpr_ty:$zm), []>, |
| Sched<[]> { |
| // Translated to the actual instructions in AArch64ISelLowering.cpp |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| class sme2_za_array_2op_multi_single_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> |
| : SMEPseudo2Instr<name, 0>, |
| Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), []> { |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| class sme2_za_array_2op_multi_multi_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, |
| SMEMatrixTypeEnum za_flag> |
| : SMEPseudo2Instr<name, 0>, |
| Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), []> { |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| class sme2_za_array_2op_multi_index_pseudo<string name, Operand index_ty, RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty, Operand imm_ty, SMEMatrixTypeEnum za_flag> |
| : SMEPseudo2Instr<name, 0>, |
| Pseudo<(outs), (ins MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm, imm_ty:$i), []> { |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME pattern match helpers. |
| //===----------------------------------------------------------------------===// |
| |
| class SME2_ZA_TwoOp_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, |
| ValueType vt, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm)>; |
| |
| |
| class SME2_ZA_TwoOp_VG2_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, |
| ValueType vt, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, (REG_SEQUENCE ZPR2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), |
| zpr_ty:$Zm)>; |
| class SME2_ZA_TwoOp_VG4_Multi_Single_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, |
| ValueType vt, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), |
| vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, |
| (REG_SEQUENCE ZPR4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), |
| zpr_ty:$Zm)>; |
| |
| class SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, |
| (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), |
| (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>; |
| |
| class SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ValueType vt, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), |
| vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm1, vt:$Zm2, vt:$Zm3, vt:$Zm4), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, |
| (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), |
| (REG_SEQUENCE ZPR4Mul4, vt:$Zm1, zsub0, vt:$Zm2, zsub1, vt:$Zm3, zsub2, vt:$Zm4, zsub3))>; |
| |
| class SME2_ZA_TwoOp_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, |
| Operand imm_ty, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn, vt:$Zm, (i32 imm_ty:$i)), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, vt:$Zn, zpr_ty:$Zm, (i32 imm_ty:$i))>; |
| |
| |
| class SME2_ZA_TwoOp_VG2_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, |
| Operand imm_ty, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), vt:$Zn1, vt:$Zn2, vt:$Zm, (i32 imm_ty:$i)), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, |
| (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), zpr_ty:$Zm, imm_ty:$i)>; |
| |
| class SME2_ZA_TwoOp_VG4_Multi_Index_Pat<string name, SDPatternOperator intrinsic, Operand index_ty, ZPRRegOp zpr_ty, ValueType vt, |
| Operand imm_ty, ComplexPattern tileslice> |
| : Pat<(intrinsic (i32 (tileslice MatrixIndexGPR32Op8_11:$base, index_ty:$offset)), |
| vt:$Zn1, vt:$Zn2, vt:$Zn3, vt:$Zn4, vt:$Zm, (i32 imm_ty:$i)), |
| (!cast<Instruction>(name # _PSEUDO) $base, $offset, |
| (REG_SEQUENCE ZPR4Mul4, vt:$Zn1, zsub0, vt:$Zn2, zsub1, vt:$Zn3, zsub2, vt:$Zn4, zsub3), |
| zpr_ty:$Zm, imm_ty:$i)>; |
| |
| class SME2_Cvt_VG4_Pat<string name, SDPatternOperator intrinsic, ValueType out_vt, ValueType in_vt> |
| : Pat<(out_vt (intrinsic in_vt:$Zn1, in_vt:$Zn2, in_vt:$Zn3, in_vt:$Zn4)), |
| (!cast<Instruction>(name) (REG_SEQUENCE ZPR4Mul4, in_vt:$Zn1, zsub0, in_vt:$Zn2, zsub1, in_vt:$Zn3, zsub2, in_vt:$Zn4, zsub3))>; |
| |
| //===----------------------------------------------------------------------===// |
| // SME Outer Products |
| //===----------------------------------------------------------------------===// |
| |
| class sme_fp_outer_product_inst<bit S, bits<2> sz, bit op, MatrixTileOperand za_ty, |
| ZPRRegOp zpr_ty, string mnemonic> |
| : I<(outs za_ty:$ZAda), |
| (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), |
| mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", |
| "", []>, |
| Sched<[]> { |
| bits<5> Zm; |
| bits<3> Pm; |
| bits<3> Pn; |
| bits<5> Zn; |
| let Inst{31-25} = 0b1000000; |
| let Inst{24} = op; |
| let Inst{23} = 0b1; |
| let Inst{22-21} = sz; |
| let Inst{20-16} = Zm; |
| let Inst{15-13} = Pm; |
| let Inst{12-10} = Pn; |
| let Inst{9-5} = Zn; |
| let Inst{4} = S; |
| let Inst{3} = op; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme_outer_product_fp32<bit S, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_fp_outer_product_inst<S, 0b00, 0b0, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<2> ZAda; |
| let Inst{1-0} = ZAda; |
| let Inst{2} = 0b0; |
| } |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), |
| (nxv4f32 ZPR32:$zn), (nxv4f32 ZPR32:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| multiclass sme_outer_product_fp64<bit S, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_fp_outer_product_inst<S, 0b10, 0b0, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<3> ZAda; |
| let Inst{2-0} = ZAda; |
| } |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), |
| (nxv2f64 ZPR64:$zn), (nxv2f64 ZPR64:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| multiclass sme2p1_fmop_tile_fp16<string mnemonic, bit bf, bit s>{ |
| def NAME : sme_fp_outer_product_inst<s, {0,bf}, 0b1, TileOp16, ZPR16, mnemonic> { |
| bits<1> ZAda; |
| let Inst{2-1} = 0b00; |
| let Inst{0} = ZAda; |
| } |
| } |
| |
| class sme_int_outer_product_inst<bits<3> opc, bit sz, bit sme2, |
| MatrixTileOperand za_ty, ZPRRegOp zpr_ty, |
| string mnemonic> |
| : I<(outs za_ty:$ZAda), |
| (ins za_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), |
| mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", |
| "", []>, |
| Sched<[]> { |
| bits<5> Zm; |
| bits<3> Pm; |
| bits<3> Pn; |
| bits<5> Zn; |
| let Inst{31-25} = 0b1010000; |
| let Inst{24} = opc{2}; // u0 |
| let Inst{23} = 0b1; |
| let Inst{22} = sz; |
| let Inst{21} = opc{1}; // u1 |
| let Inst{20-16} = Zm; |
| let Inst{15-13} = Pm; |
| let Inst{12-10} = Pn; |
| let Inst{9-5} = Zn; |
| let Inst{4} = opc{0}; //S; |
| let Inst{3} = sme2; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme_int_outer_product_i32<bits<3> opc, string mnemonic, |
| SDPatternOperator op> { |
| def NAME : sme_int_outer_product_inst<opc, 0b0, 0b0, TileOp32, |
| ZPR8, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<2> ZAda; |
| let Inst{1-0} = ZAda; |
| let Inst{2} = 0b0; |
| } |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR8, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_3:$tile, (nxv16i1 PPR3bAny:$pn), (nxv16i1 PPR3bAny:$pm), |
| (nxv16i8 ZPR8:$zn), (nxv16i8 ZPR8:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic, |
| SDPatternOperator op> { |
| def NAME : sme_int_outer_product_inst<opc, 0b1, 0b0, TileOp64, |
| ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<3> ZAda; |
| let Inst{2-0} = ZAda; |
| } |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_7:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), |
| (nxv8i16 ZPR16:$zn), (nxv8i16 ZPR16:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_7:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic> |
| : I<(outs TileOp32:$ZAda), |
| (ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm), |
| mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn, $Zm", |
| "", []>, |
| Sched<[]> { |
| bits<5> Zm; |
| bits<3> Pm; |
| bits<3> Pn; |
| bits<5> Zn; |
| bits<2> ZAda; |
| let Inst{31-25} = 0b1000000; |
| let Inst{24} = !if(opc{2}, 0, 1); |
| let Inst{23-22} = 0b10; |
| let Inst{21} = opc{1}; |
| let Inst{20-16} = Zm; |
| let Inst{15-13} = Pm; |
| let Inst{12-10} = Pn; |
| let Inst{9-5} = Zn; |
| let Inst{4} = opc{0}; |
| let Inst{3} = opc{2}; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = ZAda; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme_bf16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), |
| (nxv8bf16 ZPR16:$zn), (nxv8bf16 ZPR16:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| multiclass sme_f16_outer_product<bits<3> opc, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_outer_product_widening_inst<opc, ZPR16, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def NAME # _PSEUDO : sme_outer_product_pseudo<ZPR16, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_3:$tile, (nxv8i1 PPR3bAny:$pn), (nxv8i1 PPR3bAny:$pm), |
| (nxv8f16 ZPR16:$zn), (nxv8f16 ZPR16:$zm)), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_3:$tile, $pn, $pm, $zn, $zm)>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME Add Vector to Tile |
| //===----------------------------------------------------------------------===// |
| |
| class sme_add_vector_to_tile_inst<bit op, bit V, MatrixTileOperand tile_ty, |
| ZPRRegOp zpr_ty, string mnemonic> |
| : I<(outs tile_ty:$ZAda), |
| (ins tile_ty:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), |
| mnemonic, "\t$ZAda, $Pn/m, $Pm/m, $Zn", |
| "", []>, Sched<[]> { |
| bits<3> Pm; |
| bits<3> Pn; |
| bits<5> Zn; |
| let Inst{31-23} = 0b110000001; |
| let Inst{22} = op; |
| let Inst{21-17} = 0b01000; |
| let Inst{16} = V; |
| let Inst{15-13} = Pm; |
| let Inst{12-10} = Pn; |
| let Inst{9-5} = Zn; |
| let Inst{4-3} = 0b00; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| class sme_add_vector_to_tile_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag> |
| : Pseudo<(outs), |
| (ins i32imm:$tile, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn), []>, |
| Sched<[]> { |
| // Translated to the actual instructions in AArch64ISelLowering.cpp |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| multiclass sme_add_vector_to_tile_u32<bit V, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_add_vector_to_tile_inst<0b0, V, TileOp32, ZPR32, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<2> ZAda; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = ZAda; |
| } |
| |
| def _PSEUDO_S : sme_add_vector_to_tile_pseudo<ZPR32, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>; |
| |
| def : Pat<(op timm32_0_3:$tile, (nxv4i1 PPR3bAny:$pn), (nxv4i1 PPR3bAny:$pm), |
| (nxv4i32 ZPR32:$zn)), |
| (!cast<Instruction>(NAME # _PSEUDO_S) timm32_0_3:$tile, $pn, $pm, $zn)>; |
| } |
| |
| multiclass sme_add_vector_to_tile_u64<bit V, string mnemonic, SDPatternOperator op> { |
| def NAME : sme_add_vector_to_tile_inst<0b1, V, TileOp64, ZPR64, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<3> ZAda; |
| let Inst{2-0} = ZAda; |
| } |
| |
| def _PSEUDO_D : sme_add_vector_to_tile_pseudo<ZPR64, SMEMatrixTileD>, SMEPseudo2Instr<NAME, 0>; |
| |
| let Predicates = [HasSMEI16I64] in { |
| def : Pat<(op timm32_0_7:$tile, (nxv2i1 PPR3bAny:$pn), (nxv2i1 PPR3bAny:$pm), |
| (nxv2i64 ZPR64:$zn)), |
| (!cast<Instruction>(NAME # _PSEUDO_D) timm32_0_7:$tile, $pn, $pm, $zn)>; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME Contiguous Loads |
| //===----------------------------------------------------------------------===// |
| |
| class sme_mem_ld_ss_base<bit Q, bit V, bits<2> msz, dag outs, dag ins, |
| string mnemonic, string argstr> |
| : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<2> Rv; |
| bits<3> Pg; |
| bits<5> Rn; |
| let Inst{31-25} = 0b1110000; |
| let Inst{24} = Q; |
| let Inst{23-22} = msz; |
| let Inst{21} = 0b0; |
| let Inst{20-16} = Rm; |
| let Inst{15} = V; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = Pg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = 0b0; |
| |
| let mayLoad = 1; |
| } |
| |
| class sme_mem_ld_ss_inst<bit Q, bits<2> msz, string mnemonic, |
| MatrixTileVectorOperand tile_ty, bit is_col, |
| Operand imm_ty, RegisterOperand gpr_ty> |
| : sme_mem_ld_ss_base< |
| Q, is_col, msz, (outs tile_ty:$ZAt), |
| (ins MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, |
| gpr_ty:$Rm), |
| mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg/z, [$Rn, $Rm]">; |
| |
| multiclass sme_mem_ss_aliases_base<string mnemonic, Instruction inst, |
| MatrixTileVectorOperand tile_ty, |
| Operand imm_ty, RegisterOperand gpr_ty, |
| string pg_suffix=""> { |
| def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn, $Rm]", |
| (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, gpr_ty:$Rm), 0>; |
| // Default XZR offset aliases |
| def : InstAlias<mnemonic # "\t\\{$ZAt[$Rv, $imm]\\}, $Pg" # pg_suffix # ", [$Rn]", |
| (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 1>; |
| def : InstAlias<mnemonic # "\t$ZAt[$Rv, $imm], $Pg" # pg_suffix # ", [$Rn]", |
| (inst tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, GPR64sp:$Rn, XZR), 0>; |
| } |
| |
| multiclass sme_mem_ss_aliases<string mnemonic, string inst, bit is_col, |
| string pg_suffix=""> { |
| defm : sme_mem_ss_aliases_base<mnemonic # "b", !cast<Instruction>(inst # _B), |
| !if(is_col, TileVectorOpV8, TileVectorOpH8), |
| sme_elm_idx0_15, GPR64shifted8, pg_suffix>; |
| defm : sme_mem_ss_aliases_base<mnemonic # "h", !cast<Instruction>(inst # _H), |
| !if(is_col, TileVectorOpV16, TileVectorOpH16), |
| sme_elm_idx0_7, GPR64shifted16, pg_suffix>; |
| defm : sme_mem_ss_aliases_base<mnemonic # "w", !cast<Instruction>(inst # _S), |
| !if(is_col, TileVectorOpV32, TileVectorOpH32), |
| sme_elm_idx0_3, GPR64shifted32, pg_suffix>; |
| defm : sme_mem_ss_aliases_base<mnemonic # "d", !cast<Instruction>(inst # _D), |
| !if(is_col, TileVectorOpV64, TileVectorOpH64), |
| sme_elm_idx0_1, GPR64shifted64, pg_suffix>; |
| defm : sme_mem_ss_aliases_base<mnemonic # "q", !cast<Instruction>(inst # _Q), |
| !if(is_col, TileVectorOpV128, TileVectorOpH128), |
| sme_elm_idx0_0, GPR64shifted128, pg_suffix>; |
| } |
| |
| multiclass sme_mem_ld_ss_aliases<string inst, bit is_col> { |
| defm NAME : sme_mem_ss_aliases<"ld1", inst, is_col, "/z">; |
| } |
| |
| multiclass sme_mem_ld_ss_patterns<Instruction Inst, SDPatternOperator Load, |
| Operand tile_ty, Operand offset_ty, |
| ComplexPattern addr, |
| ComplexPattern tileslice> { |
| // base, tileslice |
| def : Pat<(Load PPR3bAny:$pg, GPR64sp:$base, tile_ty:$tile, |
| (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), |
| (Inst tile_ty:$tile, $idx, $imm, $pg, $base, XZR)>; |
| |
| // reg + reg, tileslice |
| let AddedComplexity = 1 in { |
| def : Pat<(Load PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), |
| tile_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, |
| offset_ty:$imm))), |
| (Inst tile_ty:$tile, $idx, $imm, $pg, $base, $offset)>; |
| } |
| } |
| |
| class sme_load_pseudo |
| : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, |
| i32imm:$imm, PPR3bAny:$pg, GPR64sp:$base, GPR64:$offset), []>, |
| Sched<[]> { |
| // Translated to the actual instructions in AArch64ISelLowering.cpp |
| let usesCustomInserter = 1; |
| let mayLoad = 1; |
| } |
| |
| multiclass sme_mem_ld_v_ss<string mnemonic, bit is_col> { |
| def _B : sme_mem_ld_ss_inst<0b0, 0b00, mnemonic # "b", |
| !if(is_col, TileVectorOpV8, TileVectorOpH8), |
| is_col, sme_elm_idx0_15, GPR64shifted8> { |
| bits<4> imm; |
| let Inst{3-0} = imm; |
| } |
| def _H : sme_mem_ld_ss_inst<0b0, 0b01, mnemonic # "h", |
| !if(is_col, TileVectorOpV16, TileVectorOpH16), |
| is_col, sme_elm_idx0_7, GPR64shifted16> { |
| bits<1> ZAt; |
| bits<3> imm; |
| let Inst{3} = ZAt; |
| let Inst{2-0} = imm; |
| } |
| def _S : sme_mem_ld_ss_inst<0b0, 0b10, mnemonic # "w", |
| !if(is_col, TileVectorOpV32, TileVectorOpH32), |
| is_col, sme_elm_idx0_3, GPR64shifted32> { |
| bits<2> ZAt; |
| bits<2> imm; |
| let Inst{3-2} = ZAt; |
| let Inst{1-0} = imm; |
| } |
| def _D : sme_mem_ld_ss_inst<0b0, 0b11, mnemonic # "d", |
| !if(is_col, TileVectorOpV64, TileVectorOpH64), |
| is_col, sme_elm_idx0_1, GPR64shifted64> { |
| bits<3> ZAt; |
| bits<1> imm; |
| let Inst{3-1} = ZAt; |
| let Inst{0} = imm; |
| } |
| def _Q : sme_mem_ld_ss_inst<0b1, 0b11, mnemonic # "q", |
| !if(is_col, TileVectorOpV128, TileVectorOpH128), |
| is_col, sme_elm_idx0_0, GPR64shifted128> { |
| bits<4> ZAt; |
| let Inst{3-0} = ZAt; |
| } |
| |
| defm : sme_mem_ld_ss_aliases<NAME, is_col>; |
| |
| // Pseudo instructions for lowering intrinsics, using immediates instead of |
| // tile registers. |
| def _PSEUDO_B : sme_load_pseudo; |
| def _PSEUDO_H : sme_load_pseudo; |
| def _PSEUDO_S : sme_load_pseudo; |
| def _PSEUDO_D : sme_load_pseudo; |
| def _PSEUDO_Q : sme_load_pseudo; |
| |
| defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_B), |
| !if(is_col, int_aarch64_sme_ld1b_vert, |
| int_aarch64_sme_ld1b_horiz), |
| sme_elm_idx0_0, timm32_0_15, am_sve_regreg_lsl0, |
| tileslice8>; |
| defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_H), |
| !if(is_col, int_aarch64_sme_ld1h_vert, |
| int_aarch64_sme_ld1h_horiz), |
| timm32_0_1, timm32_0_7, am_sve_regreg_lsl1, |
| tileslice16>; |
| defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_S), |
| !if(is_col, int_aarch64_sme_ld1w_vert, |
| int_aarch64_sme_ld1w_horiz), |
| timm32_0_3, timm32_0_3, am_sve_regreg_lsl2, |
| tileslice32>; |
| defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_D), |
| !if(is_col, int_aarch64_sme_ld1d_vert, |
| int_aarch64_sme_ld1d_horiz), |
| timm32_0_7, timm32_0_1, am_sve_regreg_lsl3, |
| tileslice64>; |
| defm : sme_mem_ld_ss_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| !if(is_col, int_aarch64_sme_ld1q_vert, |
| int_aarch64_sme_ld1q_horiz), |
| timm32_0_15, sme_elm_idx0_0, am_sve_regreg_lsl4, |
| tileslice128>; |
| } |
| |
| multiclass sme_mem_ld_ss<string mnemonic> { |
| defm _H : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b0>; |
| defm _V : sme_mem_ld_v_ss<mnemonic, /*is_col=*/0b1>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME Contiguous Stores |
| //===----------------------------------------------------------------------===// |
| |
| class sme_mem_st_ss_base<bit Q, bit V, bits<2> msz, dag ins, |
| string mnemonic, string argstr> |
| : I<(outs), ins, mnemonic, argstr, "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<2> Rv; |
| bits<3> Pg; |
| bits<5> Rn; |
| let Inst{31-25} = 0b1110000; |
| let Inst{24} = Q; |
| let Inst{23-22} = msz; |
| let Inst{21} = 0b1; |
| let Inst{20-16} = Rm; |
| let Inst{15} = V; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = Pg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = 0b0; |
| |
| let mayStore = 1; |
| let hasSideEffects = 1; |
| } |
| |
| class sme_mem_st_ss_inst<bit Q, bits<2> msz, string mnemonic, |
| MatrixTileVectorOperand tile_ty, bit is_col, |
| Operand imm_ty, RegisterOperand gpr_ty> |
| : sme_mem_st_ss_base< |
| Q, is_col, msz, |
| (ins tile_ty:$ZAt, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, |
| GPR64sp:$Rn, gpr_ty:$Rm), |
| mnemonic, "\t\\{$ZAt[$Rv, $imm]\\}, $Pg, [$Rn, $Rm]">; |
| |
| multiclass sme_mem_st_ss_aliases<string inst, bit is_col> { |
| defm NAME : sme_mem_ss_aliases<"st1", inst, is_col>; |
| } |
| |
| multiclass sme_mem_st_ss_patterns<Instruction Inst, SDPatternOperator Store, |
| Operand offset_ty, |
| ComplexPattern imm2tile, |
| ComplexPattern addr, |
| ComplexPattern tileslice> { |
| // base, tileslice |
| def : Pat<(Store PPR3bAny:$pg, GPR64sp:$base, (imm2tile untyped:$tile), |
| (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), |
| (Inst $tile, $idx, $imm, $pg, $base, XZR)>; |
| |
| // reg + reg, tileslice |
| let AddedComplexity = 1 in { |
| def : Pat<(Store PPR3bAny:$pg, (addr GPR64sp:$base, GPR64:$offset), |
| (imm2tile untyped:$tile), |
| (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, offset_ty:$imm))), |
| (Inst $tile, $idx, $imm, $pg, $base, $offset)>; |
| } |
| } |
| |
| multiclass sme_mem_st_v_ss<string mnemonic, bit is_col> { |
| def _B : sme_mem_st_ss_inst<0b0, 0b00, mnemonic # "b", |
| !if(is_col, TileVectorOpV8, TileVectorOpH8), |
| is_col, sme_elm_idx0_15, GPR64shifted8> { |
| bits<4> imm; |
| let Inst{3-0} = imm; |
| } |
| def _H : sme_mem_st_ss_inst<0b0, 0b01, mnemonic # "h", |
| !if(is_col, TileVectorOpV16, TileVectorOpH16), |
| is_col, sme_elm_idx0_7, GPR64shifted16> { |
| bits<1> ZAt; |
| bits<3> imm; |
| let Inst{3} = ZAt; |
| let Inst{2-0} = imm; |
| } |
| def _S : sme_mem_st_ss_inst<0b0, 0b10, mnemonic # "w", |
| !if(is_col, TileVectorOpV32, TileVectorOpH32), |
| is_col, sme_elm_idx0_3, GPR64shifted32> { |
| bits<2> ZAt; |
| bits<2> imm; |
| let Inst{3-2} = ZAt; |
| let Inst{1-0} = imm; |
| } |
| def _D : sme_mem_st_ss_inst<0b0, 0b11, mnemonic # "d", |
| !if(is_col, TileVectorOpV64, TileVectorOpH64), |
| is_col, sme_elm_idx0_1, GPR64shifted64> { |
| bits<3> ZAt; |
| bits<1> imm; |
| let Inst{3-1} = ZAt; |
| let Inst{0} = imm; |
| } |
| def _Q : sme_mem_st_ss_inst<0b1, 0b11, mnemonic # "q", |
| !if(is_col, TileVectorOpV128, TileVectorOpH128), |
| is_col, sme_elm_idx0_0, GPR64shifted128> { |
| bits<4> ZAt; |
| let Inst{3-0} = ZAt; |
| } |
| |
| defm : sme_mem_st_ss_aliases<NAME, is_col>; |
| |
| defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _B), |
| !if(is_col, int_aarch64_sme_st1b_vert, |
| int_aarch64_sme_st1b_horiz), |
| timm32_0_15, imm_to_tile8, am_sve_regreg_lsl0, |
| tileslice8>; |
| defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _H), |
| !if(is_col, int_aarch64_sme_st1h_vert, |
| int_aarch64_sme_st1h_horiz), |
| timm32_0_7, imm_to_tile16, am_sve_regreg_lsl1, |
| tileslice16>; |
| defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _S), |
| !if(is_col, int_aarch64_sme_st1w_vert, |
| int_aarch64_sme_st1w_horiz), |
| timm32_0_3, imm_to_tile32, am_sve_regreg_lsl2, |
| tileslice32>; |
| defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _D), |
| !if(is_col, int_aarch64_sme_st1d_vert, |
| int_aarch64_sme_st1d_horiz), |
| timm32_0_1, imm_to_tile64, am_sve_regreg_lsl3, |
| tileslice64>; |
| defm : sme_mem_st_ss_patterns<!cast<Instruction>(NAME # _Q), |
| !if(is_col, int_aarch64_sme_st1q_vert, |
| int_aarch64_sme_st1q_horiz), |
| sme_elm_idx0_0, imm_to_tile128, |
| am_sve_regreg_lsl4, tileslice128>; |
| } |
| |
| multiclass sme_mem_st_ss<string mnemonic> { |
| defm _H : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b0>; |
| defm _V : sme_mem_st_v_ss<mnemonic, /*is_col=*/0b1>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME Save and Restore Array |
| //===----------------------------------------------------------------------===// |
| |
| class sme_spill_fill_base<bit isStore, dag outs, dag ins, string opcodestr> |
| : I<outs, ins, opcodestr, "\t$ZAt[$Rv, $imm4], [$Rn, $offset, mul vl]", "", |
| []>, |
| Sched<[]> { |
| bits<2> Rv; |
| bits<5> Rn; |
| bits<4> imm4; |
| let Inst{31-22} = 0b1110000100; |
| let Inst{21} = isStore; |
| let Inst{20-15} = 0b000000; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b000; |
| let Inst{9-5} = Rn; |
| let Inst{4} = 0b0; |
| let Inst{3-0} = imm4; |
| } |
| |
| let mayStore = 1 in |
| class sme_spill_inst<string opcodestr> |
| : sme_spill_fill_base<0b1, (outs), |
| (ins MatrixOp:$ZAt, MatrixIndexGPR32Op12_15:$Rv, |
| sme_elm_idx0_15:$imm4, GPR64sp:$Rn, |
| imm0_15:$offset), |
| opcodestr>; |
| let mayLoad = 1 in |
| class sme_fill_inst<string opcodestr> |
| : sme_spill_fill_base<0b0, (outs MatrixOp:$ZAt), |
| (ins MatrixIndexGPR32Op12_15:$Rv, |
| sme_elm_idx0_15:$imm4, GPR64sp:$Rn, |
| imm0_15:$offset), |
| opcodestr>; |
| multiclass sme_spill<string opcodestr> { |
| def NAME : sme_spill_inst<opcodestr>; |
| def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", |
| (!cast<Instruction>(NAME) MatrixOp:$ZAt, |
| MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; |
| // base |
| def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), |
| (!cast<Instruction>(NAME) ZA, $idx, 0, $base, 0)>; |
| // scalar + immediate (mul vl) |
| let AddedComplexity = 2 in { |
| def : Pat<(int_aarch64_sme_str MatrixIndexGPR32Op12_15:$idx, |
| (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)), |
| (!cast<Instruction>(NAME) ZA, $idx, 0, $base, $imm4)>; |
| } |
| } |
| |
| multiclass sme_fill<string opcodestr> { |
| def NAME : sme_fill_inst<opcodestr>; |
| def : InstAlias<opcodestr # "\t$ZAt[$Rv, $imm4], [$Rn]", |
| (!cast<Instruction>(NAME) MatrixOp:$ZAt, |
| MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm4, GPR64sp:$Rn, 0), 1>; |
| def NAME # _PSEUDO |
| : Pseudo<(outs), |
| (ins MatrixIndexGPR32Op12_15:$idx, imm0_15:$imm4, |
| GPR64sp:$base), []>, |
| Sched<[]> { |
| // Translated to actual instruction in AArch64ISelLowering.cpp |
| let usesCustomInserter = 1; |
| let mayLoad = 1; |
| } |
| // base |
| def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, GPR64sp:$base), |
| (!cast<Instruction>(NAME # _PSEUDO) $idx, 0, $base)>; |
| // scalar + immediate (mul vl) |
| let AddedComplexity = 2 in { |
| def : Pat<(int_aarch64_sme_ldr MatrixIndexGPR32Op12_15:$idx, |
| (am_sme_indexed_b4 GPR64sp:$base, imm0_15:$imm4)), |
| (!cast<Instruction>(NAME # _PSEUDO) $idx, $imm4, $base)>; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Move instructions |
| //===----------------------------------------------------------------------===// |
| |
| class sme_vector_to_tile_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, |
| string mnemonic, string argstr> |
| : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { |
| bits<2> Rv; |
| bits<3> Pg; |
| bits<5> Zn; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-17} = 0b00000; |
| let Inst{16} = Q; |
| let Inst{15} = V; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = Pg; |
| let Inst{9-5} = Zn; |
| let Inst{4} = 0b0; |
| } |
| |
| class sme_vector_to_tile_inst<bit Q, bits<2> sz, MatrixTileVectorOperand tile_ty, |
| bit is_col, Operand imm_ty, ZPRRegOp zpr_ty, |
| string mnemonic> |
| : sme_vector_to_tile_base<Q, is_col, sz, (outs tile_ty:$ZAd), |
| (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), |
| mnemonic, "\t$ZAd[$Rv, $imm], $Pg/m, $Zn">{ |
| |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| |
| multiclass sme_vector_to_tile_aliases<Instruction inst, |
| MatrixTileVectorOperand tile_ty, |
| ZPRRegOp zpr_ty, Operand imm_ty> { |
| def : InstAlias<"mov\t$ZAd[$Rv, $imm], $Pg/m, $Zn", |
| (inst tile_ty:$ZAd, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm, PPR3bAny:$Pg, zpr_ty:$Zn), 1>; |
| } |
| |
| multiclass sme_vector_to_tile_patterns<Instruction inst, ValueType zpr_vt, |
| ValueType ppr_vt, Operand imm_ty, |
| Operand offset_ty, |
| SDPatternOperator op, |
| ComplexPattern tileslice> { |
| def : Pat<(op imm_ty:$tile, MatrixIndexGPR32Op12_15:$idx, |
| (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), |
| (inst imm_ty:$tile, $idx, 0, $pg, $zn)>; |
| let AddedComplexity = 1 in { |
| def : Pat<(op imm_ty:$tile, (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, |
| offset_ty:$imm)), |
| (ppr_vt PPR3bAny:$pg), (zpr_vt ZPRAny:$zn)), |
| (inst imm_ty:$tile, $idx, $imm, $pg, $zn)>; |
| } |
| } |
| |
| class sme_mova_insert_pseudo<SMEMatrixTypeEnum za_flag> |
| : Pseudo<(outs), (ins i32imm:$tile, MatrixIndexGPR32Op12_15:$idx, |
| i32imm:$imm, PPR3bAny:$pg, ZPRAny:$zn), []>, |
| Sched<[]> { |
| // Translated to the actual instructions in AArch64ISelLowering.cpp |
| let SMEMatrixType = za_flag; |
| let usesCustomInserter = 1; |
| } |
| |
| multiclass sme_vector_v_to_tile<string mnemonic, bit is_col> { |
| def _B : sme_vector_to_tile_inst<0b0, 0b00, !if(is_col, TileVectorOpV8, |
| TileVectorOpH8), |
| is_col, sme_elm_idx0_15, ZPR8, mnemonic>, |
| SMEPseudo2Instr<NAME # _B, 1> { |
| bits<4> imm; |
| let Inst{3-0} = imm; |
| } |
| def _H : sme_vector_to_tile_inst<0b0, 0b01, !if(is_col, TileVectorOpV16, |
| TileVectorOpH16), |
| is_col, sme_elm_idx0_7, ZPR16, mnemonic>, |
| SMEPseudo2Instr<NAME # _H, 1> { |
| bits<1> ZAd; |
| bits<3> imm; |
| let Inst{3} = ZAd; |
| let Inst{2-0} = imm; |
| } |
| def _S : sme_vector_to_tile_inst<0b0, 0b10, !if(is_col, TileVectorOpV32, |
| TileVectorOpH32), |
| is_col, sme_elm_idx0_3, ZPR32, mnemonic>, |
| SMEPseudo2Instr<NAME # _S, 1> { |
| bits<2> ZAd; |
| bits<2> imm; |
| let Inst{3-2} = ZAd; |
| let Inst{1-0} = imm; |
| } |
| def _D : sme_vector_to_tile_inst<0b0, 0b11, !if(is_col, TileVectorOpV64, |
| TileVectorOpH64), |
| is_col, sme_elm_idx0_1, ZPR64, mnemonic>, |
| SMEPseudo2Instr<NAME # _D, 1> { |
| bits<3> ZAd; |
| bits<1> imm; |
| let Inst{3-1} = ZAd; |
| let Inst{0} = imm; |
| } |
| def _Q : sme_vector_to_tile_inst<0b1, 0b11, !if(is_col, TileVectorOpV128, |
| TileVectorOpH128), |
| is_col, sme_elm_idx0_0, ZPR128, mnemonic>, |
| SMEPseudo2Instr<NAME # _Q, 1> { |
| bits<4> ZAd; |
| bits<1> imm; |
| let Inst{3-0} = ZAd; |
| } |
| |
| // Pseudo instructions for lowering intrinsics, using immediates instead of |
| // tile registers. |
| def _PSEUDO_B : sme_mova_insert_pseudo<SMEMatrixTileB>, SMEPseudo2Instr<NAME # _B, 0>; |
| def _PSEUDO_H : sme_mova_insert_pseudo<SMEMatrixTileH>, SMEPseudo2Instr<NAME # _H, 0>; |
| def _PSEUDO_S : sme_mova_insert_pseudo<SMEMatrixTileS>, SMEPseudo2Instr<NAME # _S, 0>; |
| def _PSEUDO_D : sme_mova_insert_pseudo<SMEMatrixTileD>, SMEPseudo2Instr<NAME # _D, 0>; |
| def _PSEUDO_Q : sme_mova_insert_pseudo<SMEMatrixTileQ>, SMEPseudo2Instr<NAME # _Q, 0>; |
| |
| defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _B), |
| !if(is_col, TileVectorOpV8, |
| TileVectorOpH8), |
| ZPR8, sme_elm_idx0_15>; |
| defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _H), |
| !if(is_col, TileVectorOpV16, |
| TileVectorOpH16), |
| ZPR16, sme_elm_idx0_7>; |
| defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _S), |
| !if(is_col, TileVectorOpV32, |
| TileVectorOpH32), |
| ZPR32, sme_elm_idx0_3>; |
| defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _D), |
| !if(is_col, TileVectorOpV64, |
| TileVectorOpH64), |
| ZPR64, sme_elm_idx0_1>; |
| defm : sme_vector_to_tile_aliases<!cast<Instruction>(NAME # _Q), |
| !if(is_col, TileVectorOpV128, |
| TileVectorOpH128), |
| ZPR128, sme_elm_idx0_0>; |
| |
| defvar op = !if(is_col, int_aarch64_sme_write_vert, |
| int_aarch64_sme_write_horiz); |
| |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_B), |
| nxv16i8, nxv16i1, sme_elm_idx0_0, sme_elm_idx0_15, |
| op, tileslice8>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), |
| nxv8i16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, |
| op, tileslice16>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), |
| nxv8f16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, |
| op, tileslice16>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_H), |
| nxv8bf16, nxv8i1, sme_elm_idx0_1, sme_elm_idx0_7, |
| op, tileslice16>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), |
| nxv4i32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, |
| op, tileslice32>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_S), |
| nxv4f32, nxv4i1, sme_elm_idx0_3, sme_elm_idx0_3, |
| op, tileslice32>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), |
| nxv2i64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, |
| op, tileslice64>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_D), |
| nxv2f64, nxv2i1, sme_elm_idx0_7, sme_elm_idx0_1, |
| op, tileslice64>; |
| |
| defvar opq = !if(is_col, int_aarch64_sme_writeq_vert, |
| int_aarch64_sme_writeq_horiz); |
| |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv16i8, nxv16i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv8i16, nxv8i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv8f16, nxv8i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv8bf16, nxv8i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv4i32, nxv4i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv4f32, nxv4i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv2i64, nxv2i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| defm : sme_vector_to_tile_patterns<!cast<Instruction>(NAME # _PSEUDO_Q), |
| nxv2f64, nxv2i1, sme_elm_idx0_15, |
| sme_elm_idx0_0, opq, tileslice128>; |
| } |
| |
| multiclass sme_vector_to_tile<string mnemonic> { |
| defm _H : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b0>; |
| defm _V : sme_vector_v_to_tile<mnemonic, /*is_col=*/0b1>; |
| } |
| |
| class sme_tile_to_vector_base<bit Q, bit V, bits<2> sz, dag outs, dag ins, |
| string mnemonic, string argstr> |
| : I<outs, ins, mnemonic, argstr, "", []>, Sched<[]> { |
| bits<2> Rv; |
| bits<3> Pg; |
| bits<5> Zd; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-17} = 0b00001; |
| let Inst{16} = Q; |
| let Inst{15} = V; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = Pg; |
| let Inst{9} = 0b0; |
| let Inst{4-0} = Zd; |
| } |
| |
| class sme_tile_to_vector_inst<bit Q, bits<2> sz, ZPRRegOp zpr_ty, |
| MatrixTileVectorOperand tile_ty, |
| bit is_col, Operand imm_ty, string mnemonic> |
| : sme_tile_to_vector_base<Q, is_col, sz, (outs zpr_ty:$Zd), |
| (ins zpr_ty:$_Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), |
| mnemonic, "\t$Zd, $Pg/m, $ZAn[$Rv, $imm]"> { |
| |
| let Constraints = "$Zd = $_Zd"; |
| } |
| |
| multiclass sme_tile_to_vector_aliases<Instruction inst, ZPRRegOp zpr_ty, |
| MatrixTileVectorOperand tile_ty, |
| Operand imm_ty > { |
| def : InstAlias<"mov\t$Zd, $Pg/m, $ZAn[$Rv, $imm]", |
| (inst zpr_ty:$Zd, PPR3bAny:$Pg, tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), 1>; |
| } |
| |
| multiclass sme_tile_to_vector_patterns<Instruction inst, ValueType zpr_vt, |
| ValueType ppr_vt, Operand offset_ty, |
| ComplexPattern imm2tile, |
| ComplexPattern tileslice, |
| SDPatternOperator op> { |
| def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), |
| (imm2tile untyped:$tile), MatrixIndexGPR32Op12_15:$idx)), |
| (inst $passthru, $pg, $tile, $idx, 0)>; |
| let AddedComplexity = 1 in { |
| def : Pat<(zpr_vt (op (zpr_vt ZPRAny:$passthru), (ppr_vt PPR3bAny:$pg), |
| (imm2tile untyped:$tile), |
| (i32 (tileslice MatrixIndexGPR32Op12_15:$idx, |
| offset_ty:$imm)))), |
| (inst $passthru, $pg, $tile, $idx, $imm)>; |
| } |
| } |
| |
| multiclass sme_tile_to_vector_v<string mnemonic, bit is_col> { |
| def _B : sme_tile_to_vector_inst<0b0, 0b00, ZPR8, !if(is_col, TileVectorOpV8, |
| TileVectorOpH8), |
| is_col, sme_elm_idx0_15, mnemonic> { |
| bits<4> imm; |
| let Inst{8-5} = imm; |
| } |
| def _H : sme_tile_to_vector_inst<0b0, 0b01, ZPR16, !if(is_col, TileVectorOpV16, |
| TileVectorOpH16), |
| is_col, sme_elm_idx0_7, mnemonic> { |
| bits<1> ZAn; |
| bits<3> imm; |
| let Inst{8} = ZAn; |
| let Inst{7-5} = imm; |
| } |
| def _S : sme_tile_to_vector_inst<0b0, 0b10, ZPR32, !if(is_col, TileVectorOpV32, |
| TileVectorOpH32), |
| is_col, sme_elm_idx0_3, mnemonic> { |
| bits<2> ZAn; |
| bits<2> imm; |
| let Inst{8-7} = ZAn; |
| let Inst{6-5} = imm; |
| } |
| def _D : sme_tile_to_vector_inst<0b0, 0b11, ZPR64, !if(is_col, TileVectorOpV64, |
| TileVectorOpH64), |
| is_col, sme_elm_idx0_1, mnemonic> { |
| bits<3> ZAn; |
| bits<1> imm; |
| let Inst{8-6} = ZAn; |
| let Inst{5} = imm; |
| } |
| def _Q : sme_tile_to_vector_inst<0b1, 0b11, ZPR128, !if(is_col, TileVectorOpV128, |
| TileVectorOpH128), |
| is_col, sme_elm_idx0_0, mnemonic> { |
| bits<4> ZAn; |
| let Inst{8-5} = ZAn; |
| } |
| |
| defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _B), ZPR8, |
| !if(is_col, TileVectorOpV8, |
| TileVectorOpH8), sme_elm_idx0_15>; |
| defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _H), ZPR16, |
| !if(is_col, TileVectorOpV16, |
| TileVectorOpH16), sme_elm_idx0_7>; |
| defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _S), ZPR32, |
| !if(is_col, TileVectorOpV32, |
| TileVectorOpH32), sme_elm_idx0_3>; |
| defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _D), ZPR64, |
| !if(is_col, TileVectorOpV64, |
| TileVectorOpH64), sme_elm_idx0_1>; |
| defm : sme_tile_to_vector_aliases<!cast<Instruction>(NAME # _Q), ZPR128, |
| !if(is_col, TileVectorOpV128, |
| TileVectorOpH128), sme_elm_idx0_0>; |
| |
| defvar op = !if(is_col, int_aarch64_sme_read_vert, |
| int_aarch64_sme_read_horiz); |
| |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _B), |
| nxv16i8, nxv16i1, sme_elm_idx0_15, |
| imm_to_tile8, tileslice8, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), |
| nxv8i16, nxv8i1, sme_elm_idx0_7, |
| imm_to_tile16, tileslice16, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), |
| nxv8f16, nxv8i1, sme_elm_idx0_7, |
| imm_to_tile16, tileslice16, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _H), |
| nxv8bf16, nxv8i1, sme_elm_idx0_7, |
| imm_to_tile16, tileslice16, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), |
| nxv4i32, nxv4i1, sme_elm_idx0_3, |
| imm_to_tile32, tileslice32, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _S), |
| nxv4f32, nxv4i1, sme_elm_idx0_3, |
| imm_to_tile32, tileslice32, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), |
| nxv2i64, nxv2i1, sme_elm_idx0_1, |
| imm_to_tile64, tileslice64, op>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _D), |
| nxv2f64, nxv2i1, sme_elm_idx0_1, |
| imm_to_tile64, tileslice64, op>; |
| |
| defvar opq = !if(is_col, int_aarch64_sme_readq_vert, |
| int_aarch64_sme_readq_horiz); |
| |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv16i8, nxv16i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv8i16, nxv8i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv8f16, nxv8i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv8bf16, nxv8i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv4i32, nxv4i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv4f32, nxv4i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv2i64, nxv2i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| defm : sme_tile_to_vector_patterns<!cast<Instruction>(NAME # _Q), |
| nxv2f64, nxv2i1, sme_elm_idx0_0, |
| imm_to_tile128, tileslice128, opq>; |
| } |
| |
| multiclass sme_tile_to_vector<string mnemonic> { |
| defm _H : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b0>; |
| defm _V : sme_tile_to_vector_v<mnemonic, /*is_col=*/0b1>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME Zero |
| //===----------------------------------------------------------------------===// |
| |
| // NOTE: This definition isn't really correct because there are outputs, i.e. |
| // the tile registers being zeroed. We fix this up in a custom inserter that |
| // marks the appropriate registers as being implicitly defined. |
| class sme_zero_inst<string mnemonic> |
| : I<(outs), (ins MatrixTileList:$imm), |
| mnemonic, "\t$imm", "", []>, Sched<[]> { |
| bits<8> imm; |
| let Inst{31-8} = 0b110000000000100000000000; |
| let Inst{7-0} = imm; |
| } |
| |
| multiclass sme_zero<string mnemonic> { |
| def NAME : sme_zero_inst<mnemonic>; |
| |
| def : InstAlias<"zero\t\\{za\\}", (!cast<Instruction>(NAME) 0b11111111), 1>; |
| def : InstAlias<"zero\t\\{za0.h\\}", (!cast<Instruction>(NAME) 0b01010101), 1>; |
| def : InstAlias<"zero\t\\{za1.h\\}", (!cast<Instruction>(NAME) 0b10101010), 1>; |
| def : InstAlias<"zero\t\\{za0.s\\}", (!cast<Instruction>(NAME) 0b00010001), 1>; |
| def : InstAlias<"zero\t\\{za1.s\\}", (!cast<Instruction>(NAME) 0b00100010), 1>; |
| def : InstAlias<"zero\t\\{za2.s\\}", (!cast<Instruction>(NAME) 0b01000100), 1>; |
| def : InstAlias<"zero\t\\{za3.s\\}", (!cast<Instruction>(NAME) 0b10001000), 1>; |
| def : InstAlias<"zero\t\\{za0.s,za1.s\\}", (!cast<Instruction>(NAME) 0b00110011), 1>; |
| def : InstAlias<"zero\t\\{za0.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10011001), 1>; |
| def : InstAlias<"zero\t\\{za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01100110), 1>; |
| def : InstAlias<"zero\t\\{za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11001100), 1>; |
| def : InstAlias<"zero\t\\{za0.s,za1.s,za2.s\\}", (!cast<Instruction>(NAME) 0b01110111), 1>; |
| def : InstAlias<"zero\t\\{za0.s,za1.s,za3.s\\}", (!cast<Instruction>(NAME) 0b10111011), 1>; |
| def : InstAlias<"zero\t\\{za0.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11011101), 1>; |
| def : InstAlias<"zero\t\\{za1.s,za2.s,za3.s\\}", (!cast<Instruction>(NAME) 0b11101110), 1>; |
| |
| def NAME # _PSEUDO : Pseudo<(outs), (ins i32imm:$tilelist), []>, |
| Sched<[]> { |
| // Translated to the actual instructions in AArch64ISelLowering.cpp |
| let usesCustomInserter = 1; |
| } |
| |
| def : Pat<(int_aarch64_sme_zero timm32_0_255:$imm), |
| (!cast<Instruction>(NAME # _PSEUDO) timm32_0_255:$imm)>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SVE2 Instructions |
| //===----------------------------------------------------------------------===// |
| |
| class sve2_int_perm_revd<string asm> |
| : I<(outs ZPR128:$Zd), (ins ZPR128:$_Zd, PPR3bAny:$Pg, ZPR128:$Zn), |
| asm, "\t$Zd, $Pg/m, $Zn", "", []>, |
| Sched<[]> { |
| bits<5> Zd; |
| bits<3> Pg; |
| bits<5> Zn; |
| let Inst{31-24} = 0b00000101; |
| let Inst{23-22} = 0b00; // size |
| let Inst{21-13} = 0b101110100; |
| let Inst{12-10} = Pg; |
| let Inst{9-5} = Zn; |
| let Inst{4-0} = Zd; |
| |
| let Constraints = "$Zd = $_Zd"; |
| let DestructiveInstType = DestructiveUnary; |
| let ElementSize = ZPR128.ElementSize; |
| } |
| |
| multiclass sve2_int_perm_revd<string asm, SDPatternOperator op> { |
| def NAME : sve2_int_perm_revd<asm>; |
| |
| def : SVE_1_Op_Passthru_Pat<nxv16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME)>; |
| def : SVE_1_Op_Passthru_Pat<nxv8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME)>; |
| def : SVE_1_Op_Passthru_Pat<nxv4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME)>; |
| def : SVE_1_Op_Passthru_Pat<nxv2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME)>; |
| } |
| |
| class sve2_clamp<string asm, bits<2> sz, bit U, ZPRRegOp zpr_ty> |
| : I<(outs zpr_ty:$Zd), (ins zpr_ty:$Zn, zpr_ty:$Zm, zpr_ty:$_Zd), |
| asm, "\t$Zd, $Zn, $Zm", "", []>, |
| Sched<[]> { |
| bits<5> Zm; |
| bits<5> Zn; |
| bits<5> Zd; |
| let Inst{31-24} = 0b01000100; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b0; |
| let Inst{20-16} = Zm; |
| let Inst{15-11} = 0b11000; |
| let Inst{10} = U; |
| let Inst{9-5} = Zn; |
| let Inst{4-0} = Zd; |
| |
| let Constraints = "$Zd = $_Zd"; |
| let DestructiveInstType = DestructiveOther; |
| let ElementSize = zpr_ty.ElementSize; |
| } |
| |
| multiclass sve2_clamp<string asm, bit U, SDPatternOperator op> { |
| def _B : sve2_clamp<asm, 0b00, U, ZPR8>; |
| def _H : sve2_clamp<asm, 0b01, U, ZPR16>; |
| def _S : sve2_clamp<asm, 0b10, U, ZPR32>; |
| def _D : sve2_clamp<asm, 0b11, U, ZPR64>; |
| |
| def : SVE_3_Op_Pat<nxv16i8, op, nxv16i8, nxv16i8, nxv16i8, !cast<Instruction>(NAME # _B)>; |
| def : SVE_3_Op_Pat<nxv8i16, op, nxv8i16, nxv8i16, nxv8i16, !cast<Instruction>(NAME # _H)>; |
| def : SVE_3_Op_Pat<nxv4i32, op, nxv4i32, nxv4i32, nxv4i32, !cast<Instruction>(NAME # _S)>; |
| def : SVE_3_Op_Pat<nxv2i64, op, nxv2i64, nxv2i64, nxv2i64, !cast<Instruction>(NAME # _D)>; |
| } |
| |
| class sve2_int_perm_sel_p<string asm, PPRRegOp ppr_ty, Operand imm_ty> |
| : I<(outs PPRAny:$Pd), (ins PPRAny:$Pn, ppr_ty:$Pm, |
| MatrixIndexGPR32Op12_15:$Rv, imm_ty:$imm), |
| asm, "\t$Pd, $Pn, $Pm[$Rv, $imm]", "", []>, |
| Sched<[]> { |
| bits<2> Rv; |
| bits<4> Pn; |
| bits<4> Pm; |
| bits<4> Pd; |
| let Inst{31-24} = 0b00100101; |
| let Inst{21} = 0b1; |
| let Inst{17-16} = Rv; |
| let Inst{15-14} = 0b01; |
| let Inst{13-10} = Pn; |
| let Inst{9} = 0b0; |
| let Inst{8-5} = Pm; |
| let Inst{4} = 0b0; |
| let Inst{3-0} = Pd; |
| } |
| |
| multiclass sve2_int_perm_sel_p<string asm, SDPatternOperator op> { |
| def _B : sve2_int_perm_sel_p<asm, PPR8, sme_elm_idx0_15> { |
| bits<4> imm; |
| let Inst{23-22} = imm{3-2}; |
| let Inst{20-19} = imm{1-0}; |
| let Inst{18} = 0b1; |
| } |
| def _H : sve2_int_perm_sel_p<asm, PPR16, sme_elm_idx0_7> { |
| bits<3> imm; |
| let Inst{23-22} = imm{2-1}; |
| let Inst{20} = imm{0}; |
| let Inst{19-18} = 0b10; |
| } |
| def _S : sve2_int_perm_sel_p<asm, PPR32, sme_elm_idx0_3> { |
| bits<2> imm; |
| let Inst{23-22} = imm{1-0}; |
| let Inst{20-18} = 0b100; |
| } |
| def _D : sve2_int_perm_sel_p<asm, PPR64, sme_elm_idx0_1> { |
| bits<1> imm; |
| let Inst{23} = imm; |
| let Inst{22} = 0b1; |
| let Inst{20-18} = 0b000; |
| } |
| |
| def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", |
| (!cast<Instruction>(NAME # _B) PNRAny:$Pd, |
| PNRAny:$Pn, PPR8:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_15:$imm), 0>; |
| def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", |
| (!cast<Instruction>(NAME # _H) PNRAny:$Pd, |
| PNRAny:$Pn, PPR16:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_7:$imm), 0>; |
| def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", |
| (!cast<Instruction>(NAME # _S) PNRAny:$Pd, |
| PNRAny:$Pn, PPR32:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_3:$imm), 0>; |
| def : InstAlias<asm # "\t$Pd, $Pn, $Pm[$Rv, $imm]", |
| (!cast<Instruction>(NAME # _D) PNRAny:$Pd, |
| PNRAny:$Pn, PPR64:$Pm, MatrixIndexGPR32Op12_15:$Rv, sme_elm_idx0_1:$imm), 0>; |
| |
| def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm), |
| MatrixIndexGPR32Op12_15:$idx)), |
| (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, 0)>; |
| def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm), |
| MatrixIndexGPR32Op12_15:$idx)), |
| (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, 0)>; |
| def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm), |
| MatrixIndexGPR32Op12_15:$idx)), |
| (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, 0)>; |
| def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm), |
| MatrixIndexGPR32Op12_15:$idx)), |
| (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, 0)>; |
| |
| let AddedComplexity = 1 in { |
| def : Pat<(nxv16i1 (op (nxv16i1 PPRAny:$Pn), (nxv16i1 PPRAny:$Pm), |
| (i32 (tileslice8 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_15:$imm)))), |
| (!cast<Instruction>(NAME # _B) $Pn, $Pm, $idx, $imm)>; |
| def : Pat<(nxv8i1 (op (nxv8i1 PPRAny:$Pn), (nxv8i1 PPRAny:$Pm), |
| (i32 (tileslice16 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_7:$imm)))), |
| (!cast<Instruction>(NAME # _H) $Pn, $Pm, $idx, $imm)>; |
| def : Pat<(nxv4i1 (op (nxv4i1 PPRAny:$Pn), (nxv4i1 PPRAny:$Pm), |
| (i32 (tileslice32 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_3:$imm)))), |
| (!cast<Instruction>(NAME # _S) $Pn, $Pm, $idx, $imm)>; |
| def : Pat<(nxv2i1 (op (nxv2i1 PPRAny:$Pn), (nxv2i1 PPRAny:$Pm), |
| (i32 (tileslice64 MatrixIndexGPR32Op12_15:$idx, sme_elm_idx0_1:$imm)))), |
| (!cast<Instruction>(NAME # _D) $Pn, $Pm, $idx, $imm)>; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Instructions |
| //===----------------------------------------------------------------------===// |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 single-multi ternary int/fp, two/four registers |
| |
| class sme2_dot_mla_add_sub_array_vg24_single<bits<7> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAd), |
| (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, |
| sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), |
| mnemonic,"\t$ZAd[$Rv, $imm3, " # !if(op{5}, "vgx4", "vgx2") # "], $Zn, $Zm", |
| "", []> , Sched<[]> { |
| bits<4> Zm; |
| bits<5> Zn; |
| bits<2> Rv; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = op{6}; //sz |
| let Inst{21} = 0b1; |
| let Inst{20} = op{5}; //vgx4 |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = op{4-2}; |
| let Inst{9-5} = Zn; |
| let Inst{4-3} = op{1-0}; |
| let Inst{2-0} = imm3; |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| multiclass sme2_dot_mla_add_sub_array_vg24_single<string mnemonic, bits<7> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty>{ |
| def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; |
| } |
| |
| multiclass sme2_dot_mla_add_sub_array_vg2_single<string mnemonic, bits<7> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ |
| def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; |
| } |
| |
| multiclass sme2_dot_mla_add_sub_array_vg4_single<string mnemonic, bits<7> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty, ValueType vty, SDPatternOperator intrinsic>{ |
| def NAME: sme2_dot_mla_add_sub_array_vg24_single<op, matrix_ty, multi_vector_ty, zpr_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, zpr_ty, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, vty, tileslice16>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 multiple vectors ternary INT/FP two and four registers |
| class sme2_dot_mla_add_sub_array_vg2_multi<bits<6> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAd), |
| (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, |
| sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), |
| mnemonic, "\t$ZAd[$Rv, $imm3, vgx2], $Zn, $Zm", |
| "", []>, Sched<[]>{ |
| bits<4> Zm; |
| bits<4> Zn; |
| bits<2> Rv; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000011; |
| let Inst{22} = op{5}; //sz |
| let Inst{21} = 0b1; |
| let Inst{20-17} = Zm; |
| let Inst{16-15} = 0b00; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = op{4-2}; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{4-3} = op{1-0}; |
| let Inst{2-0} = imm3; |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| multiclass sme2_dot_mla_add_sub_array_vg2_multi<string mnemonic, bits<6> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, ValueType zpr_ty, |
| SDPatternOperator intrinsic> { |
| def NAME : sme2_dot_mla_add_sub_array_vg2_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; |
| } |
| |
| class sme2_dot_mla_add_sub_array_vg4_multi<bits<6> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAd), |
| (ins matrix_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, |
| sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), |
| mnemonic, "\t$ZAd[$Rv, $imm3, vgx4], $Zn, $Zm", |
| "", []>, Sched<[]>{ |
| bits<3> Zm; |
| bits<3> Zn; |
| bits<2> Rv; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000011; |
| let Inst{22} = op{5}; //sz |
| let Inst{21} = 0b1; |
| let Inst{20-18} = Zm; |
| let Inst{17-15} = 0b010; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = op{4-2}; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = 0b00; |
| let Inst{4-3} = op{1-0}; |
| let Inst{2-0} = imm3; |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| multiclass sme2_dot_mla_add_sub_array_vg4_multi<string mnemonic, bits<6> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ValueType zpr_ty, SDPatternOperator intrinsic>{ |
| def NAME : sme2_dot_mla_add_sub_array_vg4_multi<op, matrix_ty, multi_vector_ty, mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME, intrinsic, sme_elm_idx0_7, zpr_ty, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm3], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, multi_vector_ty:$Zn, multi_vector_ty:$Zm), 0>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 multiple vectors binary two or four registers |
| |
| class sme2_multivec_accum_add_sub<string mnemonic, bit sz, bit vg4, bits<3> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> |
| : I<(outs matrix_ty:$ZAdn), |
| (ins matrix_ty:$_ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), |
| mnemonic, "\t$ZAdn[$Rv, $imm3, " # !if(vg4, "vgx4", "vgx2") # "], $Zm", |
| "", []>, Sched<[]> { |
| bits<2> Rv; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000011; |
| let Inst{22} = sz; |
| let Inst{21-19} = 0b100; |
| let Inst{18} = op{2}; |
| let Inst{17} = 0b0; |
| let Inst{16} = vg4; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b111; |
| let Inst{5} = 0b0; |
| let Inst{4-3} = op{1-0}; |
| let Inst{2-0} = imm3; |
| |
| let Constraints = "$ZAdn = $_ZAdn"; |
| } |
| |
| class sme2_multivec_accum_add_sub_vg2<string mnemonic, bit sz, bits<3> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> |
| : sme2_multivec_accum_add_sub<mnemonic, sz, 0b0, op, matrix_ty, vector_ty> { |
| bits<4> Zm; |
| let Inst{9-6} = Zm; |
| } |
| |
| |
| multiclass sme2_multivec_accum_add_sub_vg2<string mnemonic, bits<4> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> { |
| def NAME : sme2_multivec_accum_add_sub_vg2<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>; |
| |
| def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; |
| } |
| |
| class sme2_multivec_accum_add_sub_vg4<string mnemonic, bit sz, bits<3> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> |
| : sme2_multivec_accum_add_sub<mnemonic, sz, 0b1, op, matrix_ty, vector_ty> { |
| bits<3> Zm; |
| let Inst{9-7} = Zm; |
| let Inst{6} = 0b0; |
| } |
| |
| multiclass sme2_multivec_accum_add_sub_vg4<string mnemonic, bits<4> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> { |
| def NAME : sme2_multivec_accum_add_sub_vg4<mnemonic, op{3}, op{2-0}, matrix_ty, vector_ty>; |
| |
| def : InstAlias<mnemonic # "\t$ZAdn[$Rv, $imm3], $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAdn, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, vector_ty:$Zm), 0>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Multi-vector - Multiple and Single SVE Destructive |
| // Two and Four registers |
| |
| class sme2_sve_destructive_vector_vg2_single<bits<2> sz, bits<7> op, |
| RegisterOperand vector_ty, |
| ZPRRegOp zpr_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), |
| mnemonic, "\t$Zdn, $_Zdn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<4> Zdn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-20} = 0b10; |
| let Inst{19-16} = Zm; |
| let Inst{15-11} = 0b10100; |
| let Inst{10-5} = op{6-1}; |
| let Inst{4-1} = Zdn; |
| let Inst{0} = op{0}; |
| |
| let Constraints = "$Zdn = $_Zdn"; |
| } |
| |
| multiclass sme2_fp_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { |
| def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; |
| } |
| |
| multiclass sme2_int_sve_destructive_vector_vg2_single<string mnemonic, bits<7> op> { |
| def _B : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_b_mul_r, ZPR4b8, mnemonic>; |
| def _H : sme2_sve_destructive_vector_vg2_single<0b01, op, ZZ_h_mul_r, ZPR4b16, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg2_single<0b10, op, ZZ_s_mul_r, ZPR4b32, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg2_single<0b11, op, ZZ_d_mul_r, ZPR4b64, mnemonic>; |
| } |
| |
| // SME2.1 fmax/fmin instructions. |
| multiclass sme2p1_bf_max_min_vector_vg2_single<string mnemonic, bits<7>op> { |
| def _H : sme2_sve_destructive_vector_vg2_single<0b00, op, ZZ_h_mul_r, |
| ZPR4b16, mnemonic>; |
| } |
| |
| class sme2_sve_destructive_vector_vg4_single<bits<2> sz, bits<7> op, |
| RegisterOperand vector_ty, |
| ZPRRegOp zpr_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, zpr_ty:$Zm), |
| mnemonic, "\t$Zdn, $_Zdn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<3> Zdn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-20} = 0b10; |
| let Inst{19-16} = Zm; |
| let Inst{15-11} = 0b10101; |
| let Inst{10-5} = op{6-1}; |
| let Inst{4-2} = Zdn; |
| let Inst{1} = 0b0; |
| let Inst{0} = op{0}; |
| |
| let Constraints = "$Zdn = $_Zdn"; |
| } |
| |
| multiclass sme2_fp_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { |
| def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; |
| } |
| |
| multiclass sme2_int_sve_destructive_vector_vg4_single<string mnemonic, bits<7> op> { |
| def _B : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_b_mul_r, ZPR4b8, mnemonic>; |
| def _H : sme2_sve_destructive_vector_vg4_single<0b01, op, ZZZZ_h_mul_r, ZPR4b16, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg4_single<0b10, op, ZZZZ_s_mul_r, ZPR4b32, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg4_single<0b11, op, ZZZZ_d_mul_r, ZPR4b64, mnemonic>; |
| } |
| |
| // SME2.1 fmax/fmin instructions. |
| multiclass sme2p1_bf_max_min_vector_vg4_single<string mnemonic, bits<7>op> { |
| def _H : sme2_sve_destructive_vector_vg4_single<0b00, op, ZZZZ_h_mul_r, |
| ZPR4b16, mnemonic>; |
| } |
| |
| class sme2_sve_destructive_vector_vg2_multi<bits<2> sz, bits<7> op, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), |
| mnemonic, "\t$Zdn, $_Zdn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<4> Zdn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| let Inst{20-17} = Zm; |
| let Inst{16-11} = 0b010110; |
| let Inst{10-5} = op{6-1}; |
| let Inst{4-1} = Zdn; |
| let Inst{0} = op{0}; |
| |
| let Constraints = "$Zdn = $_Zdn"; |
| } |
| |
| multiclass sme2_fp_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { |
| def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; |
| } |
| |
| multiclass sme2_int_sve_destructive_vector_vg2_multi<string mnemonic, bits<7> op> { |
| def _B : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_b_mul_r, mnemonic>; |
| def _H : sme2_sve_destructive_vector_vg2_multi<0b01, op, ZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg2_multi<0b10, op, ZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg2_multi<0b11, op, ZZ_d_mul_r, mnemonic>; |
| } |
| |
| // SME2.1 fmax/fmin instructions. |
| multiclass sme2p1_bf_max_min_vector_vg2_multi<string mnemonic, bits<7>op> { |
| def _H : sme2_sve_destructive_vector_vg2_multi<0b00, op, ZZ_h_mul_r, |
| mnemonic>; |
| } |
| |
| class sme2_sve_destructive_vector_vg4_multi<bits<2> sz, bits<7> op, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zdn), (ins vector_ty:$_Zdn, vector_ty:$Zm), |
| mnemonic, "\t$Zdn, $_Zdn, $Zm", |
| "", []>, Sched<[]> { |
| bits<3> Zm; |
| bits<3> Zdn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| let Inst{20-18} = Zm; |
| let Inst{17-11} = 0b0010111; |
| let Inst{10-5} = op{6-1}; |
| let Inst{4-2} = Zdn; |
| let Inst{1} = 0b0; |
| let Inst{0} = op{0}; |
| |
| let Constraints = "$Zdn = $_Zdn"; |
| } |
| |
| multiclass sme2_fp_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { |
| def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; |
| } |
| |
| multiclass sme2_int_sve_destructive_vector_vg4_multi<string mnemonic, bits<7> op> { |
| def _B : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_b_mul_r, mnemonic>; |
| def _H : sme2_sve_destructive_vector_vg4_multi<0b01, op, ZZZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sve_destructive_vector_vg4_multi<0b10, op, ZZZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sve_destructive_vector_vg4_multi<0b11, op, ZZZZ_d_mul_r, mnemonic>; |
| } |
| |
| // SME2.1 fmax/fmin instructions. |
| multiclass sme2p1_bf_max_min_vector_vg4_multi<string mnemonic, bits<7>op> { |
| def _H : sme2_sve_destructive_vector_vg4_multi<0b00, op, ZZZZ_h_mul_r, |
| mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Multi-vector - Index/Single/Multi Array Vectors FMA sources |
| |
| class sme2_mla_long_array_index_base<bits<2> op0, bits<2> op, Operand index_ty, |
| RegisterOperand multi_vector_ty, |
| string mnemonic, string vg_acronym=""> |
| : I<(outs MatrixOp32:$ZAda), |
| (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm, multi_vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), |
| mnemonic, "\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm$i3", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = op0; |
| let Inst{21} = 0b0; |
| let Inst{20} = !if(!eq(vg_acronym, ""), 0, 1); |
| let Inst{19-16} = Zm; |
| let Inst{14-13} = Rv; |
| let Inst{12} = 0b1; |
| let Inst{4-3} = op; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_mla_long_array_index<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_index_base<op0, op, uimm3s2range, ZPR16, |
| mnemonic>, SMEPseudo2Instr<NAME # _S, 1> { |
| bits<3> i3; |
| bits<5> Zn; |
| bits<3> imm; |
| let Inst{15} = i3{2}; |
| let Inst{11-10} = i3{1-0}; |
| let Inst{9-5} = Zn; |
| let Inst{2-0} = imm; |
| } |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_Multi_Index_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange3s2>; |
| } |
| |
| class sme2_mla_long_array_vg2_index<string mnemonic, bits<2> op0, bits<2> op> |
| : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZ_h_mul_r, |
| mnemonic, "vgx2"> { |
| bits<3> i3; |
| bits<4> Zn; |
| bits<2> imm; |
| let Inst{15} = 0b0; |
| let Inst{11-10} = i3{2-1}; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{2} = i3{0}; |
| let Inst{1-0} = imm; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg2_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg2_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg2_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; |
| } |
| |
| class sme2_mla_long_array_vg4_index<string mnemonic, bits<2> op0, bits<2> op> |
| : sme2_mla_long_array_index_base<op0, op, uimm2s2range, ZZZZ_h_mul_r, |
| mnemonic, "vgx4"> { |
| bits<3> i3; |
| bits<3> Zn; |
| bits<2> imm; |
| let Inst{15} = 0b1; |
| let Inst{11-10} = i3{2-1}; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = 0b00; |
| let Inst{2} = i3{0}; |
| let Inst{1-0} = imm; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg4_index<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, VectorIndexH32b_timm, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg4_index<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg4_index<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, ZPR4b16, VectorIndexH32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, VectorIndexH32b_timm, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i3", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH32b_timm:$i3), 0>; |
| } |
| |
| class sme2_mla_long_array<bits<2>op0, bits<2> op, Operand index_ty, |
| RegisterOperand first_vector_ty, |
| RegisterOperand second_vector_ty, |
| string mnemonic, string vg_acronym=""> |
| : I<(outs MatrixOp32:$ZAda), |
| (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, |
| index_ty:$imm, first_vector_ty:$Zn, second_vector_ty:$Zm), |
| mnemonic,"\t$ZAda[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn, $Zm", |
| "", []> , Sched<[]> { |
| bits<2> Rv; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = op0; |
| let Inst{21} = 0b1; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-11} = 0b01; |
| let Inst{10} = !if(!eq(vg_acronym, ""), 1, 0); |
| let Inst{4-3} = op; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_mla_long_array_single<string mnemonic, bits<2> op0, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array<op0, op, uimm3s2range, ZPR16, ZPR4b16, |
| mnemonic> , SMEPseudo2Instr<NAME # _S, 1>{ |
| bits<4> Zm; |
| bits<5> Zn; |
| bits<3> imm; |
| let Inst{20} = 0b0; |
| let Inst{19-16} = Zm; |
| let Inst{9-5} = Zn; |
| let Inst{2-0} = imm; |
| } |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm3s2range, ZPR16, ZPR4b16, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_Multi_Single_Pat<NAME # _S, intrinsic, uimm3s2range, ZPR4b16, zpr_ty, tileslicerange3s2>; |
| } |
| |
| class sme2_mla_long_array_vg24_single<bits<2> op0, bit vg4, bits<2> op, |
| RegisterOperand first_vector_ty, |
| string mnemonic, string vg_acronym> |
| : sme2_mla_long_array<op0, op, uimm2s2range, first_vector_ty, ZPR4b16, |
| mnemonic, vg_acronym> { |
| bits<4> Zm; |
| bits<5> Zn; |
| bits<2> imm; |
| let Inst{20} = vg4; |
| let Inst{19-16} = Zm; |
| let Inst{9-5} = Zn; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = imm; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg2_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg24_single<0b00, 0b0, op, ZZ_h, mnemonic, |
| "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg2_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg24_single<0b01, 0b0, op, ZZ_h, mnemonic, |
| "vgx2">, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZ_h, ZPR4b16, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h:$Zn, ZPR4b16:$Zm), 0>; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg4_single<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg24_single<0b00, 0b1, op, ZZZZ_h, mnemonic, |
| "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, zpr_ty, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg4_single<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg24_single<0b01, 0b1, op, ZZZZ_h, mnemonic, |
| "vgx4">, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_single_pseudo<NAME # _S, uimm2s2range, ZZZZ_h, ZPR4b16, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Single_Pat<NAME # _S, intrinsic, uimm2s2range, ZPR4b16, nxv8i16, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h:$Zn, ZPR4b16:$Zm), 0>; |
| } |
| class sme2_mla_long_array_vg2_multi<string mnemonic, bits<2> op0, bits<2> op> |
| : sme2_mla_long_array<op0, op, uimm2s2range, ZZ_h_mul_r, ZZ_h_mul_r, mnemonic, |
| "vgx2"> { |
| bits<4> Zm; |
| bits<4> Zn; |
| bits<2> imm; |
| let Inst{20-17} = Zm; |
| let Inst{16} = 0b0; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = imm; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg2_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg2_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg2_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZ_h_mul_r, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZ_h_mul_r:$Zn, ZZ_h_mul_r:$Zm), 0>; |
| } |
| |
| class sme2_mla_long_array_vg4_multi<string mnemonic, bits<2> op0, bits<2> op> |
| : sme2_mla_long_array<op0, op, uimm2s2range, ZZZZ_h_mul_r, ZZZZ_h_mul_r, mnemonic, |
| "vgx4"> { |
| bits<3> Zm; |
| bits<3> Zn; |
| bits<2> imm; |
| let Inst{20-18} = Zm; |
| let Inst{17} = 0b0; |
| let Inst{16} = 0b1; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = 0b00; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = imm; |
| } |
| |
| multiclass sme2_fp_mla_long_array_vg4_multi<string mnemonic, bits<2> op, ValueType zpr_ty, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b10, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, zpr_ty, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; |
| } |
| |
| multiclass sme2_int_mla_long_array_vg4_multi<string mnemonic, bits<2> op, SDPatternOperator intrinsic> { |
| def _S : sme2_mla_long_array_vg4_multi<mnemonic, 0b11, op>, SMEPseudo2Instr<NAME # _S, 1>; |
| |
| def _S_PSEUDO : sme2_za_array_2op_multi_multi_pseudo<NAME # _S, uimm2s2range, ZZZZ_h_mul_r, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Multi_Pat<NAME # _S, intrinsic, uimm2s2range, nxv8i16, tileslicerange2s2>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm2], $Zn, $Zm", |
| (!cast<Instruction>(NAME #_S) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s2range:$imm2, ZZZZ_h_mul_r:$Zn, ZZZZ_h_mul_r:$Zm), 0>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| class sme2_frint_cvt_vg2_multi<bits<2>sz, bits<5>op, RegisterOperand first_ty, |
| RegisterOperand second_ty, string mnemonic> |
| : I<(outs first_ty:$Zd), (ins second_ty:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<4> Zn; |
| bits<4> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-20} = 0b10; |
| let Inst{19-16} = op{4-1}; |
| let Inst{15-10} = 0b111000; |
| let Inst{9-6} = Zn; |
| let Inst{5} = op{0}; |
| let Inst{4-1} = Zd; |
| let Inst{0} = 0b0; |
| } |
| |
| // SME2 multi-vec FP to int convert two registers |
| // SME2 multi-vec int to FP two registers |
| multiclass sme2_fp_cvt_vg2_multi<string mnemonic, bits<5> op> { |
| def NAME : sme2_frint_cvt_vg2_multi<0b00, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; |
| } |
| |
| // SME2 multi-vec FRINT two registers |
| multiclass sme2_frint_vector_vg2_multi<string mnemonic, bits<5> op> { |
| def _S : sme2_frint_cvt_vg2_multi<0b10, op, ZZ_s_mul_r, ZZ_s_mul_r, mnemonic>; |
| } |
| |
| class sme2_frint_zip_cvt_vg4_multi<bits<2>sz, bits<7>op, RegisterOperand first_ty, |
| RegisterOperand second_ty, string mnemonic> |
| : I<(outs first_ty:$Zd), (ins second_ty:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<3> Zn; |
| bits<3> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-20} = 0b11; |
| let Inst{19-16} = op{6-3}; |
| let Inst{15-10} = 0b111000; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = op{2-1}; |
| let Inst{4-2} = Zd; |
| let Inst{1} = op{0}; |
| let Inst{0} = 0b0; |
| } |
| |
| // SME2 multi-vec FP to int convert four registers |
| // SME2 multi-vec int to FP four registers |
| multiclass sme2_fp_cvt_vg4_multi<string mnemonic, bits<7> op> { |
| def NAME : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, mnemonic>; |
| } |
| |
| // SME2 multi-vec quadwords ZIP four registers |
| multiclass sme2_zip_vector_vg4<string mnemonic, bits<7> op> { |
| def _B : sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_b_mul_r, ZZZZ_b_mul_r, |
| mnemonic>; |
| def _H : sme2_frint_zip_cvt_vg4_multi<0b01, op, ZZZZ_h_mul_r, ZZZZ_h_mul_r, |
| mnemonic>; |
| def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, |
| mnemonic>; |
| def _D : sme2_frint_zip_cvt_vg4_multi<0b11, op, ZZZZ_d_mul_r, ZZZZ_d_mul_r, |
| mnemonic>; |
| } |
| |
| // SME2 multi-vec quadwords ZIP four registers |
| multiclass sme2_zip_vector_vg4_Q<string mnemonic, bits<7> op> { |
| def NAME: sme2_frint_zip_cvt_vg4_multi<0b00, op, ZZZZ_q_mul_r, ZZZZ_q_mul_r, |
| mnemonic>; |
| } |
| |
| // SME2 multi-vec FRINT four registers |
| multiclass sme2_frint_vector_vg4_multi<string mnemonic, bits<7> op> { |
| def _S : sme2_frint_zip_cvt_vg4_multi<0b10, op, ZZZZ_s_mul_r, ZZZZ_s_mul_r, |
| mnemonic>; |
| } |
| |
| class sme2_cvt_vg2_single<string mnemonic, bits<4> op> |
| : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<4> Zn; |
| bits<5> Zd; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = op{3}; |
| let Inst{21-18} = 0b1000; |
| let Inst{17-16} = op{2-1}; |
| let Inst{15-10} = 0b111000; |
| let Inst{9-6} = Zn; |
| let Inst{5} = op{0}; |
| let Inst{4-0} = Zd; |
| } |
| |
| // SME2 multi-vec FP down convert two registers |
| // SME2 multi-vec int down convert two registers |
| multiclass sme2_cvt_vg2_single<string mnemonic, bits<4> op, ValueType out_vt, |
| ValueType in_vt, SDPatternOperator intrinsic> { |
| def NAME : sme2_cvt_vg2_single<mnemonic, op>; |
| def : SVE2p1_Cvt_VG2_Pat<NAME, intrinsic, out_vt, in_vt>; |
| } |
| |
| class sme2_cvt_unpk_vector_vg2<bits<2>sz, bits<3> op, bit u, RegisterOperand first_ty, |
| RegisterOperand second_ty, string mnemonic> |
| : I<(outs first_ty:$Zd), (ins second_ty:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<4> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-19} = 0b100; |
| let Inst{18-16} = op; |
| let Inst{15-10} = 0b111000; |
| let Inst{9-5} = Zn; |
| let Inst{4-1} = Zd; |
| let Inst{0} = u; |
| } |
| |
| // SME2 multi-vec unpack two registers |
| multiclass sme2_unpk_vector_vg2<string mnemonic, bit u> { |
| def _H : sme2_cvt_unpk_vector_vg2<0b01, 0b101, u, ZZ_h_mul_r, ZPR8, mnemonic>; |
| def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b101, u, ZZ_s_mul_r, ZPR16, mnemonic>; |
| def _D : sme2_cvt_unpk_vector_vg2<0b11, 0b101, u, ZZ_d_mul_r, ZPR32, mnemonic>; |
| } |
| |
| // SME2.1 multi-vec convert two registers |
| multiclass sme2p1_fp_cvt_vector_vg2_single<string mnemonic, bit l> { |
| def _S : sme2_cvt_unpk_vector_vg2<0b10, 0b000, l, ZZ_s_mul_r, ZPR16, mnemonic>; |
| } |
| |
| class sme2_cvt_vg4_single<bit sz, bits<3> op, RegisterOperand first_ty, |
| RegisterOperand second_ty, string mnemonic> |
| : I<(outs first_ty:$Zd), (ins second_ty:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<3> Zn; |
| bits<5> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23} = sz; |
| let Inst{22} = op{2}; |
| let Inst{21-10} = 0b110011111000; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = op{1-0}; |
| let Inst{4-0} = Zd; |
| } |
| |
| // SME2 multi-vec int down convert four registers |
| multiclass sme2_int_cvt_vg4_single<string mnemonic, bits<3> op, SDPatternOperator intrinsic> { |
| def _StoB : sme2_cvt_vg4_single<0, op, ZPR8, ZZZZ_s_mul_r, mnemonic>; |
| def _DtoH : sme2_cvt_vg4_single<1, op, ZPR16, ZZZZ_d_mul_r, mnemonic>; |
| |
| def : SME2_Cvt_VG4_Pat<NAME # _StoB, intrinsic, nxv16i8, nxv4i32>; |
| def : SME2_Cvt_VG4_Pat<NAME # _DtoH, intrinsic, nxv8i16, nxv2i64>; |
| } |
| |
| class sme2_unpk_vector_vg4<bits<2>sz, bit u, RegisterOperand first_ty, |
| RegisterOperand second_ty, string mnemonic> |
| : I<(outs first_ty:$Zd), (ins second_ty:$Zn), |
| mnemonic, "\t$Zd, $Zn", "", []>, Sched<[]> { |
| bits<4> Zn; |
| bits<3> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21-10} = 0b110101111000; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{4-2} = Zd; |
| let Inst{1} = 0b0; |
| let Inst{0} = u; |
| } |
| |
| // SME2 multi-vec UNPK four registers |
| multiclass sme2_unpk_vector_vg4<string mnemonic, bit u> { |
| def _H : sme2_unpk_vector_vg4<0b01, u, ZZZZ_h_mul_r, ZZ_b_mul_r, mnemonic>; |
| def _S : sme2_unpk_vector_vg4<0b10, u, ZZZZ_s_mul_r, ZZ_h_mul_r, mnemonic>; |
| def _D : sme2_unpk_vector_vg4<0b11, u, ZZZZ_d_mul_r, ZZ_s_mul_r, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 multi-vec CLAMP registers |
| |
| class sme2_clamp_vector_vg24_multi<bits<2> sz, bits<3> op1, bit u, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, string mnemonic> |
| : I<(outs multi_vector_ty:$Zd), |
| (ins multi_vector_ty:$_Zd, vector_ty:$Zn, vector_ty:$Zm), |
| mnemonic, "\t$Zd, $Zn, $Zm", |
| "", []>, Sched<[]>{ |
| bits<5> Zm; |
| bits<5> Zn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| let Inst{20-16} = Zm; |
| let Inst{15-13} = 0b110; |
| let Inst{12-10} = op1; |
| let Inst{9-5} = Zn; |
| let Inst{0} = u; |
| |
| let Constraints = "$Zd = $_Zd"; |
| } |
| |
| class sme2_clamp_vector_vg2_multi<bits<2> sz, bits<3> op1, bit u, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, string mnemonic> |
| : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, |
| mnemonic>{ |
| bits<4> Zd; |
| let Inst{4-1} = Zd; |
| } |
| |
| multiclass sme2_fp_clamp_vector_vg2_multi<string mnemonic>{ |
| def _H : sme2_clamp_vector_vg2_multi<0b01, 0b000, 0b0, ZZ_h_mul_r, ZPR16, mnemonic>; |
| def _S : sme2_clamp_vector_vg2_multi<0b10, 0b000, 0b0, ZZ_s_mul_r, ZPR32, mnemonic>; |
| def _D : sme2_clamp_vector_vg2_multi<0b11, 0b000, 0b0, ZZ_d_mul_r, ZPR64, mnemonic>; |
| } |
| |
| multiclass sme2_int_clamp_vector_vg2_multi<string mnemonic, bit u>{ |
| def _B : sme2_clamp_vector_vg2_multi<0b00, 0b001, u, ZZ_b_mul_r, ZPR8, mnemonic>; |
| def _H : sme2_clamp_vector_vg2_multi<0b01, 0b001, u, ZZ_h_mul_r, ZPR16, mnemonic>; |
| def _S : sme2_clamp_vector_vg2_multi<0b10, 0b001, u, ZZ_s_mul_r, ZPR32, mnemonic>; |
| def _D : sme2_clamp_vector_vg2_multi<0b11, 0b001, u, ZZ_d_mul_r, ZPR64, mnemonic>; |
| } |
| |
| // SME2.1 multi-vec FCLAMP two registers |
| multiclass sme2p1_bfclamp_vector_vg2_multi<string mnemonic> { |
| def _H : sme2_clamp_vector_vg2_multi<0b00, 0b000, 0b0, ZZ_h_mul_r, ZPR16, |
| mnemonic>; |
| } |
| |
| class sme2_clamp_vector_vg4_multi<bits<2> sz, bits<3> op1, bit u, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, string mnemonic> |
| : sme2_clamp_vector_vg24_multi<sz, op1, u, multi_vector_ty, vector_ty, |
| mnemonic>{ |
| bits<3> Zd; |
| let Inst{4-2} = Zd; |
| let Inst{1} = 0b0; |
| } |
| |
| multiclass sme2_fp_clamp_vector_vg4_multi<string mnemonic>{ |
| def _H : sme2_clamp_vector_vg4_multi<0b01, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, mnemonic>; |
| def _S : sme2_clamp_vector_vg4_multi<0b10, 0b010, 0b0, ZZZZ_s_mul_r, ZPR32, mnemonic>; |
| def _D : sme2_clamp_vector_vg4_multi<0b11, 0b010, 0b0, ZZZZ_d_mul_r, ZPR64, mnemonic>; |
| } |
| |
| multiclass sme2_int_clamp_vector_vg4_multi<string mnemonic, bit u>{ |
| def _B : sme2_clamp_vector_vg4_multi<0b00, 0b011, u, ZZZZ_b_mul_r, ZPR8, mnemonic>; |
| def _H : sme2_clamp_vector_vg4_multi<0b01, 0b011, u, ZZZZ_h_mul_r, ZPR16, mnemonic>; |
| def _S : sme2_clamp_vector_vg4_multi<0b10, 0b011, u, ZZZZ_s_mul_r, ZPR32, mnemonic>; |
| def _D : sme2_clamp_vector_vg4_multi<0b11, 0b011, u, ZZZZ_d_mul_r, ZPR64, mnemonic>; |
| } |
| |
| // SME2.1 multi-vec FCLAMP four registers |
| multiclass sme2p1_bfclamp_vector_vg4_multi<string mnemonic> { |
| def _H : sme2_clamp_vector_vg4_multi<0b00, 0b010, 0b0, ZZZZ_h_mul_r, ZPR16, |
| mnemonic>; |
| } |
| |
| // SME2 multi-vec ZIP two registers |
| class sme2_zip_vector_vg2<bits<2> sz, bit q, bit u, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, string mnemonic> |
| : I<(outs multi_vector_ty:$Zd), (ins vector_ty:$Zn, vector_ty:$Zm), |
| mnemonic, "\t$Zd, $Zn, $Zm", |
| "", []>, Sched<[]>{ |
| bits<4> Zd; |
| bits<5> Zm; |
| bits<5> Zn; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| let Inst{20-16} = Zm; |
| let Inst{15-11} = 0b11010; |
| let Inst{10} = q; |
| let Inst{9-5} = Zn; |
| let Inst{4-1} = Zd; |
| let Inst{0} = u; |
| } |
| |
| multiclass sme2_zip_vector_vg2<string mnemonic, bit op> { |
| def _B : sme2_zip_vector_vg2<0b00, 0b0, op, ZZ_b_mul_r, ZPR8, mnemonic>; |
| def _H : sme2_zip_vector_vg2<0b01, 0b0, op, ZZ_h_mul_r, ZPR16, mnemonic>; |
| def _S : sme2_zip_vector_vg2<0b10, 0b0, op, ZZ_s_mul_r, ZPR32, mnemonic>; |
| def _D : sme2_zip_vector_vg2<0b11, 0b0, op, ZZ_d_mul_r, ZPR64, mnemonic>; |
| def _Q : sme2_zip_vector_vg2<0b00, 0b1, op, ZZ_q_mul_r, ZPR128, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Dot Products and MLA |
| |
| class sme2_multi_vec_array_vg2_index<bit sz, bits<6> op, MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, Operand index_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<4> Zn; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = sz; |
| let Inst{21-20} = 0b01; |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = op{5-3}; |
| let Inst{9-6} = Zn; |
| let Inst{5-3} = op{2-0}; |
| let Inst{2-0} = imm3; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| // SME2 multi-vec ternary indexed two registers 32-bit |
| multiclass sme2_multi_vec_array_vg2_index_32b<string mnemonic, bits<4> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, ValueType vt, |
| SDPatternOperator intrinsic> { |
| def NAME : sme2_multi_vec_array_vg2_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, vector_ty, |
| VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<2> i; |
| let Inst{11-10} = i; |
| } |
| |
| def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; |
| } |
| |
| // SME2.1 multi-vec ternary indexed two registers 16-bit |
| multiclass sme2p1_multi_vec_array_vg2_index_16b<string mnemonic, bits<2> op> { |
| def NAME : sme2_multi_vec_array_vg2_index<0b0, {0b1,?,?,op,?}, MatrixOp16, |
| ZZ_h_mul_r, ZPR4b16, |
| VectorIndexH, mnemonic> { |
| bits<3> i; |
| let Inst{11-10} = i{2-1}; |
| let Inst{3} = i{0}; |
| } |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; |
| } |
| |
| // SME2 multi-vec ternary indexed two registers 64-bit |
| |
| class sme2_multi_vec_array_vg2_index_64b<bits<2> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, |
| string mnemonic> |
| : I<(outs MatrixOp64:$ZAda), |
| (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), |
| mnemonic, "\t$ZAda[$Rv, $imm3, vgx2], $Zn, $Zm$i1", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<1> i1; |
| bits<4> Zn; |
| bits<3> imm3; |
| let Inst{31-20} = 0b110000011101; |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-11} = 0b00; |
| let Inst{10} = i1; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{4-3} = op; |
| let Inst{2-0} = imm3; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_multi_vec_array_vg2_index_64b<string mnemonic, bits<2> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, ValueType vt, |
| SDPatternOperator intrinsic> { |
| def NAME : sme2_multi_vec_array_vg2_index_64b<op, multi_vector_ty, vector_ty, |
| mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG2_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexD32b_timm, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", |
| (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; |
| } |
| |
| class sme2_multi_vec_array_vg4_index<bit sz, bits<6> op, MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, Operand index_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, index_ty:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<3> Zn; |
| bits<3> imm3; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = sz; |
| let Inst{21-20} = 0b01; |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b1; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = op{5-3}; |
| let Inst{9-7} = Zn; |
| let Inst{6} = 0b0; |
| let Inst{5-3} = op{2-0}; |
| let Inst{2-0} = imm3; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| // SME2 multi-vec ternary indexed four registers 32-bit |
| multiclass sme2_multi_vec_array_vg4_index_32b<string mnemonic, bits<4> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, ValueType vt, |
| SDPatternOperator intrinsic> { |
| def NAME : sme2_multi_vec_array_vg4_index<0b1, {op{3},?,?,op{2-0}}, MatrixOp32, multi_vector_ty, |
| vector_ty, VectorIndexS32b_timm, mnemonic>, SMEPseudo2Instr<NAME, 1> { |
| bits<2> i; |
| let Inst{11-10} = i; |
| } |
| |
| def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexS32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vt, VectorIndexS32b_timm, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexS32b_timm:$i), 0>; |
| } |
| |
| // SME2.1 multi-vec ternary indexed four registers 16-bit |
| multiclass sme2p1_multi_vec_array_vg4_index_16b<string mnemonic, bits<2> op> { |
| def NAME : sme2_multi_vec_array_vg4_index<0b0,{0b1,?,?,op,?}, MatrixOp16, |
| ZZZZ_h_mul_r, ZPR4b16, |
| VectorIndexH, mnemonic>{ |
| bits<3> i; |
| let Inst{11-10} = i{2-1}; |
| let Inst{3} = i{0}; |
| } |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp16:$ZAda, MatrixIndexGPR32Op8_11:$Rv, |
| sme_elm_idx0_7:$imm3, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; |
| } |
| |
| // SME2 multi-vec ternary indexed four registers 64-bit |
| class sme2_multi_vec_array_vg4_index_64b<bits<3> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, |
| string mnemonic> |
| : I<(outs MatrixOp64:$ZAda), |
| (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), |
| mnemonic, "\t$ZAda[$Rv, $imm3, vgx4], $Zn, $Zm$i1", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<1> i1; |
| bits<3> Zn; |
| bits<3> imm3; |
| let Inst{31-20} = 0b110000011101; |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b1; |
| let Inst{14-13} = Rv; |
| let Inst{12} = 0b0; |
| let Inst{11} = op{2}; |
| let Inst{10} = i1; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = 0b00; |
| let Inst{4-3} = op{1-0}; |
| let Inst{2-0} = imm3; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_multi_vec_array_vg4_index_64b<string mnemonic, bits<3> op, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp vector_ty, ValueType vty, |
| SDPatternOperator intrinsic> { |
| def NAME : sme2_multi_vec_array_vg4_index_64b<op, multi_vector_ty, vector_ty, |
| mnemonic>, SMEPseudo2Instr<NAME, 1>; |
| |
| def _PSEUDO : sme2_za_array_2op_multi_index_pseudo<NAME, sme_elm_idx0_7, multi_vector_ty, vector_ty, VectorIndexD32b_timm, SMEMatrixArray>; |
| |
| def : SME2_ZA_TwoOp_VG4_Multi_Index_Pat<NAME, intrinsic, sme_elm_idx0_7, vector_ty, vty, VectorIndexD32b_timm, tileslice16>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm3], $Zn, $Zm$i1", |
| (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, sme_elm_idx0_7:$imm3, |
| multi_vector_ty:$Zn, vector_ty:$Zm, VectorIndexD32b_timm:$i1), 0>; |
| } |
| //===----------------------------------------------------------------------===// |
| // SME2 multi-vec indexed long long MLA one source 32-bit |
| class sme2_mla_ll_array_index_32b<string mnemonic, bits<3> op> |
| : I<(outs MatrixOp32:$ZAda), |
| (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR8:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<4> i; |
| bits<5> Zn; |
| bits<2> imm2; |
| let Inst{31-20} = 0b110000010000; |
| let Inst{19-16} = Zm; |
| let Inst{15} = i{3}; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = i{2-0}; |
| let Inst{9-5} = Zn; |
| let Inst{4-2} = op; |
| let Inst{1-0} = imm2; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| // SME2 multi-vec indexed long long MLA one source 64-bit |
| |
| class sme2_mla_ll_array_index_64b<string mnemonic, bits<2> op> |
| : I<(outs MatrixOp64:$ZAda), |
| (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm2, ZPR16:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm2], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<3> i; |
| bits<5> Zn; |
| bits<2> imm2; |
| let Inst{31-20} = 0b110000011000; |
| let Inst{19-16} = Zm; |
| let Inst{15} = i{2}; |
| let Inst{14-13} = Rv; |
| let Inst{12} = 0b0; |
| let Inst{11-10} = i{1-0}; |
| let Inst{9-5} = Zn; |
| let Inst{4-3} = op; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = imm2; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| class sme2_mla_ll_array_vg24_index_32b<bit vg4, bits<3> op, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs MatrixOp32:$ZAda), |
| (ins MatrixOp32:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, |
| vector_ty:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<4> i; |
| bit imm; |
| let Inst{31-20} = 0b110000010001; |
| let Inst{19-16} = Zm; |
| let Inst{15} = vg4; |
| let Inst{14-13} = Rv; |
| let Inst{12} = 0b0; |
| let Inst{11-10} = i{3-2}; |
| let Inst{5-3} = op; |
| let Inst{2-1} = i{1-0}; |
| let Inst{0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| //SME2 multi-vec indexed long long MLA two sources 32-bit |
| |
| multiclass sme2_mla_ll_array_vg2_index_32b<string mnemonic, bits<3> op> { |
| def NAME: sme2_mla_ll_array_vg24_index_32b<0b0, op, ZZ_b_mul_r, mnemonic> { |
| bits<4> Zn; |
| let Inst{9-6} = Zn; |
| } |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; |
| } |
| |
| // SME2 multi-vec indexed long long MLA four sources 32-bit |
| |
| multiclass sme2_mla_ll_array_vg4_index_32b<string mnemonic, bits<3> op> { |
| def NAME: sme2_mla_ll_array_vg24_index_32b<0b1, op, ZZZZ_b_mul_r, mnemonic> { |
| bits<3> Zn; |
| let Inst{9-7} = Zn; |
| let Inst{6} = 0b0; |
| } |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp32:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_b_mul_r:$Zn, ZPR4b8:$Zm, VectorIndexB:$i), 0>; |
| } |
| class sme2_mla_ll_array_vg24_index_64b<bit vg4, bits<2> op, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs MatrixOp64:$ZAda), |
| (ins MatrixOp64:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, |
| vector_ty:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), |
| mnemonic, "\t$ZAda[$Rv, $imm, " # !if(vg4, "vgx4", "vgx2") # "], $Zn, $Zm$i", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<3> i; |
| bit imm; |
| let Inst{31-20} = 0b110000011001; |
| let Inst{19-16} = Zm; |
| let Inst{15} = vg4; |
| let Inst{14-13} = Rv; |
| let Inst{12-11} = 0b00; |
| let Inst{10} = i{2}; |
| let Inst{5} = 0b0; |
| let Inst{4-3} = op; |
| let Inst{2-1} = i{1-0}; |
| let Inst{0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| // SME2 multi-vec indexed long long MLA two sources 64-bit |
| |
| multiclass sme2_mla_ll_array_vg2_index_64b<string mnemonic, bits<2> op> { |
| def NAME: sme2_mla_ll_array_vg24_index_64b<0b0, op, ZZ_h_mul_r, mnemonic>{ |
| bits<4> Zn; |
| let Inst{9-6} = Zn; |
| } |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; |
| } |
| |
| // SME2 multi-vec indexed long long MLA four sources 64-bit |
| |
| multiclass sme2_mla_ll_array_vg4_index_64b<string mnemonic, bits<2> op> { |
| def NAME: sme2_mla_ll_array_vg24_index_64b<0b1, op, ZZZZ_h_mul_r, mnemonic>{ |
| bits<3> Zn; |
| let Inst{9-7} = Zn; |
| let Inst{6} = 0b0; |
| } |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm$i", |
| (!cast<Instruction>(NAME) MatrixOp64:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, ZZZZ_h_mul_r:$Zn, ZPR4b16:$Zm, VectorIndexH:$i), 0>; |
| } |
| |
| |
| //SME2 multiple and single vector long long FMA one source |
| |
| class sme2_mla_ll_array_single<string mnemonic, bits<4> op, |
| MatrixOperand matrix_ty, ZPRRegOp vector_ty, |
| ZPRRegOp zpr_ty> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm2s4range:$imm, |
| vector_ty:$Zn, zpr_ty:$Zm), |
| mnemonic, "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<5> Zn; |
| bits<2> imm; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = op{3}; //sz |
| let Inst{21-20} = 0b10; |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b001; |
| let Inst{9-5} = Zn; |
| let Inst{4-2} = op{2-0}; |
| let Inst{1-0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| class sme2_mla_ll_array_vg24_single<bits<5> op, MatrixOperand matrix_ty, |
| RegisterOperand vector_ty, ZPRRegOp zpr_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, |
| vector_ty:$Zn, zpr_ty:$Zm), |
| mnemonic, "\t$ZAda[$Rv, $imm, " # !if(op{3}, "vgx4", "vgx2") # "], $Zn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<5> Zn; |
| bit imm; |
| let Inst{31-23} = 0b110000010; |
| let Inst{22} = op{4}; //sz |
| let Inst{21} = 0b1; |
| let Inst{20} = op{3}; //vg4 |
| let Inst{19-16} = Zm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b000; |
| let Inst{9-5} = Zn; |
| let Inst{4-2} = op{2-0}; |
| let Inst{1} = 0b0; |
| let Inst{0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| //SME2 single-multi long long MLA two and four sources |
| |
| multiclass sme2_mla_ll_array_vg24_single<string mnemonic, bits<5> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand multi_vector_ty, |
| ZPRRegOp zpr_ty> { |
| def NAME: sme2_mla_ll_array_vg24_single<op, matrix_ty, multi_vector_ty, |
| zpr_ty, mnemonic>; |
| |
| def : InstAlias<mnemonic # "\t$ZAd[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAd, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, multi_vector_ty:$Zn, zpr_ty:$Zm), 0>; |
| } |
| |
| // SME2 multiple vectors long long MLA two sources |
| |
| class sme2_mla_ll_array_vg2_multi<bits<4> op, MatrixOperand matrix_ty, |
| RegisterOperand vector_ty,string mnemonic> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, |
| vector_ty:$Zn, vector_ty:$Zm), |
| mnemonic, "\t$ZAda[$Rv, $imm, vgx2], $Zn, $Zm", |
| "", []>, Sched<[]> { |
| bits<4> Zm; |
| bits<2> Rv; |
| bits<4> Zn; |
| bit imm; |
| let Inst{31-23} = 0b110000011; |
| let Inst{22} = op{3}; // sz |
| let Inst{21} = 0b1; |
| let Inst{20-17} = Zm; |
| let Inst{16-15} = 0b00; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b000; |
| let Inst{9-6} = Zn; |
| let Inst{5} = 0b0; |
| let Inst{4-2} = op{2-0}; |
| let Inst{1} = 0b0; |
| let Inst{0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_mla_ll_array_vg2_multi<string mnemonic, bits<4> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> { |
| def NAME : sme2_mla_ll_array_vg2_multi<op, matrix_ty, vector_ty, mnemonic>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; |
| } |
| |
| // SME2 multiple vectors long long MLA four sources |
| |
| class sme2_mla_ll_array_vg4_multi<bits<4> op,MatrixOperand matrix_ty, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs matrix_ty:$ZAda), |
| (ins matrix_ty:$_ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, |
| vector_ty:$Zn, vector_ty:$Zm), |
| mnemonic, "\t$ZAda[$Rv, $imm, vgx4], $Zn, $Zm", |
| "", []>, Sched<[]> { |
| bits<3> Zm; |
| bits<2> Rv; |
| bits<3> Zn; |
| bit imm; |
| let Inst{31-23} = 0b110000011; |
| let Inst{22} = op{3}; // sz |
| let Inst{21} = 0b1; |
| let Inst{20-18} = Zm; |
| let Inst{17-15} = 0b010; |
| let Inst{14-13} = Rv; |
| let Inst{12-10} = 0b000; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = 0b00; |
| let Inst{4-2} = op{2-0}; |
| let Inst{1} = 0b0; |
| let Inst{0} = imm; |
| |
| let Constraints = "$ZAda = $_ZAda"; |
| } |
| |
| multiclass sme2_mla_ll_array_vg4_multi<string mnemonic, bits<4> op, |
| MatrixOperand matrix_ty, |
| RegisterOperand vector_ty> { |
| def NAME : sme2_mla_ll_array_vg4_multi<op, matrix_ty, vector_ty, mnemonic>; |
| |
| def : InstAlias<mnemonic # "\t$ZAda[$Rv, $imm], $Zn, $Zm", |
| (!cast<Instruction>(NAME) matrix_ty:$ZAda, MatrixIndexGPR32Op8_11:$Rv, uimm1s4range:$imm, vector_ty:$Zn, vector_ty:$Zm), 0>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Outer Product and Accumulate |
| |
| multiclass sme2_int_mopx_tile<string mnemonic, bits<3> op> { |
| def NAME : sme_int_outer_product_inst<op, 0b0, 0b1, TileOp32, ZPR16, mnemonic> { |
| bits<2> ZAda; |
| let Inst{1-0} = ZAda; |
| let Inst{2} = 0b0; |
| } |
| } |
| |
| multiclass sme2_bfp_mopx_tile<string mnemonic, bits<3> op> { |
| def NAME : sme_outer_product_widening_inst<op, ZPR32, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===/// |
| // SME2 Zero Lookup Table. |
| class sme2_zero_zt<string mnemonic, bits<4> opc> |
| : I<(outs ZTR:$ZT), (ins ), |
| mnemonic, "\t\\{ $ZT \\}", |
| "", []>, Sched<[]> { |
| let Inst{31-4} = 0b1100000001001000000000000000; |
| let Inst{3-0} = opc; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 lookup table load/store |
| class sme2_spill_fill_vector<string mnemonic, bits<8> opc> |
| : I<!if(opc{7}, (outs ), (outs ZTR:$ZTt)), |
| !if(opc{7}, (ins ZTR:$ZTt, GPR64sp:$Rn), (ins GPR64sp:$Rn)), |
| mnemonic, "\t$ZTt, [$Rn]", |
| "", []>, Sched<[]> { |
| bits<5> Rn; |
| let Inst{31-22} = 0b1110000100; |
| let Inst{21-16} = opc{7-2}; |
| let Inst{15-10} = 0b100000; |
| let Inst{9-5} = Rn; |
| let Inst{4-2} = 0b000; |
| let Inst{1-0} = opc{1-0}; |
| |
| let mayLoad = !not(opc{7}); |
| let mayStore = opc{7}; |
| } |
| |
| //===----------------------------------------------------------------------===/// |
| // SME2 move to/from lookup table |
| class sme2_movt_zt_to_scalar<string mnemonic, bits<7> opc> |
| : I<(outs GPR64:$Rt), (ins ZTR:$ZTt, uimm3s8:$imm3), |
| mnemonic, "\t$Rt, $ZTt$imm3", |
| "", []>, Sched<[]> { |
| bits<3> imm3; |
| bits<5> Rt; |
| let Inst{31-15} = 0b11000000010011000; |
| let Inst{14-12} = imm3; |
| let Inst{11-5} = opc; |
| let Inst{4-0} = Rt; |
| } |
| |
| class sme2_movt_scalar_to_zt<string mnemonic, bits<7> opc> |
| : I<(outs ZTR:$ZTt), (ins uimm3s8:$imm3, GPR64:$Rt), |
| mnemonic, "\t$ZTt$imm3, $Rt", |
| "", []>, Sched<[]> { |
| bits<3> imm3; |
| bits<5> Rt; |
| let Inst{31-15} = 0b11000000010011100; |
| let Inst{14-12} = imm3; |
| let Inst{11-5} = opc; |
| let Inst{4-0} = Rt; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 lookup table expand one register |
| class sme2_luti_vector_index<bits<2> sz, bits<7> opc, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, string mnemonic> |
| : I<(outs vector_ty:$Zd), |
| (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), |
| mnemonic, "\t$Zd, $ZTt, $Zn$i", |
| "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<5> Zd; |
| let Inst{31-19} = 0b1100000011001; |
| let Inst{18-14} = opc{6-2}; |
| let Inst{13-12} = sz; |
| let Inst{11-10} = opc{1-0}; |
| let Inst{9-5} = Zn; |
| let Inst{4-0} = Zd; |
| } |
| |
| class sme2_luti2_vector_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_index<sz, {1,?,?,?,?,0,0}, vector_ty, VectorIndexB, mnemonic> { |
| bits<4> i; |
| let Inst{17-14} = i; |
| } |
| |
| multiclass sme2_luti2_vector_index<string mnemonic> { |
| def _B : sme2_luti2_vector_index<0b00, ZPR8, mnemonic>; |
| def _H : sme2_luti2_vector_index<0b01, ZPR16, mnemonic>; |
| def _S : sme2_luti2_vector_index<0b10, ZPR32, mnemonic>; |
| } |
| |
| class sme2_luti4_vector_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_index<sz, {0,1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { |
| bits<3> i; |
| let Inst{16-14} = i; |
| } |
| |
| multiclass sme2_luti4_vector_index<string mnemonic> { |
| def _B : sme2_luti4_vector_index<0b00, ZPR8, mnemonic>; |
| def _H : sme2_luti4_vector_index<0b01, ZPR16, mnemonic>; |
| def _S : sme2_luti4_vector_index<0b10, ZPR32, mnemonic>; |
| } |
| |
| // SME2 lookup table expand two contiguous registers |
| class sme2_luti_vector_vg2_index<bits<2> sz, bits<6> opc, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, string mnemonic> |
| : I<(outs vector_ty:$Zd), |
| (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), |
| mnemonic, "\t$Zd, $ZTt, $Zn$i", |
| "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<4> Zd; |
| let Inst{31-19} = 0b1100000010001; |
| let Inst{18-15} = opc{5-2}; |
| let Inst{14} = 0b1; |
| let Inst{13-12} = sz; |
| let Inst{11-10} = opc{1-0}; |
| let Inst{9-5} = Zn; |
| let Inst{4-1} = Zd; |
| let Inst{0} = 0b0; |
| } |
| |
| class sme2_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_vg2_index<sz, {1,?,?,?,0,0}, vector_ty, VectorIndexH, mnemonic> { |
| bits<3> i; |
| let Inst{17-15} = i; |
| } |
| |
| multiclass sme2_luti2_vector_vg2_index<string mnemonic> { |
| def _B : sme2_luti2_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; |
| def _H : sme2_luti2_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; |
| def _S : sme2_luti2_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; |
| } |
| |
| class sme2_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_vg2_index<sz, {0,1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { |
| bits<2> i; |
| let Inst{16-15} = i; |
| } |
| |
| multiclass sme2_luti4_vector_vg2_index<string mnemonic> { |
| def _B : sme2_luti4_vector_vg2_index<0b00, ZZ_b_mul_r, mnemonic>; |
| def _H : sme2_luti4_vector_vg2_index<0b01, ZZ_h_mul_r, mnemonic>; |
| def _S : sme2_luti4_vector_vg2_index<0b10, ZZ_s_mul_r, mnemonic>; |
| } |
| |
| // SME2 lookup table expand four contiguous registers |
| class sme2_luti_vector_vg4_index<bits<2> sz, bits<5>opc, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, string mnemonic> |
| : I<(outs vector_ty:$Zd), |
| (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), |
| mnemonic, "\t$Zd, $ZTt, $Zn$i", |
| "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<3> Zd; |
| let Inst{31-19} = 0b1100000010001; |
| let Inst{18-16} = opc{4-2}; |
| let Inst{15-14} = 0b10; |
| let Inst{13-12} = sz; |
| let Inst{11-10} = opc{1-0}; |
| let Inst{9-5} = Zn; |
| let Inst{4-2} = Zd; |
| let Inst{1-0} = 0b00; |
| } |
| |
| class sme2_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_vg4_index<sz, {1,?,?,0,0}, vector_ty, VectorIndexS, mnemonic> { |
| bits<2> i; |
| let Inst{17-16} = i; |
| } |
| |
| multiclass sme2_luti2_vector_vg4_index<string mnemonic> { |
| def _B : sme2_luti2_vector_vg4_index<0b00, ZZZZ_b_mul_r, mnemonic>; |
| def _H : sme2_luti2_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; |
| def _S : sme2_luti2_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; |
| } |
| |
| class sme2_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_luti_vector_vg4_index<sz, {0,1,?,0,0}, vector_ty, VectorIndexD, mnemonic> { |
| bits<1> i; |
| let Inst{16} = i; |
| } |
| |
| multiclass sme2_luti4_vector_vg4_index<string mnemonic> { |
| def _H : sme2_luti4_vector_vg4_index<0b01, ZZZZ_h_mul_r, mnemonic>; |
| def _S : sme2_luti4_vector_vg4_index<0b10, ZZZZ_s_mul_r, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 MOV |
| class sme2_mova_vec_to_tile_vg2_multi_base<bits<2> sz, bit v, |
| RegisterOperand tile_ty, |
| Operand index_ty, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs tile_ty:$ZAd), |
| (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, vector_ty:$Zn), |
| mnemonic, "\t$ZAd[$Rs, $imm], $Zn", |
| "", []>, Sched<[]> { |
| bits<2> Rs; |
| bits<4> Zn; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-16} = 0b000100; |
| let Inst{15} = v; |
| let Inst{14-13} = Rs; |
| let Inst{12-10} = 0b000; |
| let Inst{9-6} = Zn; |
| let Inst{5-3} = 0b000; |
| |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| multiclass sme2_mova_vec_to_tile_or_array_aliases<int prefer, Instruction inst, |
| RegisterOperand tile_or_array_ty, |
| RegisterOperand rv_ty, |
| Operand index_ty, |
| RegisterOperand vector_ty, |
| string mnemonic, |
| string vg_acronym=""> { |
| def : InstAlias<mnemonic # "\t$ZAd[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "], $Zn", |
| (inst tile_or_array_ty:$ZAd, rv_ty:$Rs, index_ty:$imm, vector_ty:$Zn), prefer>; |
| |
| } |
| |
| // SME2 move vector to tile, two registers |
| multiclass sme2_mova_vec_to_tile_vg2_multi_base<bit v, string mnemonic> { |
| |
| def _B : sme2_mova_vec_to_tile_vg2_multi_base<0b00, v, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| uimm3s2range, ZZ_b_mul_r, |
| mnemonic> { |
| bits<3> imm; |
| let Inst{2-0} = imm; |
| } |
| |
| def _H : sme2_mova_vec_to_tile_vg2_multi_base<0b01, v, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| uimm2s2range, ZZ_h_mul_r, |
| mnemonic> { |
| bits<1> ZAd; |
| bits<2> imm; |
| let Inst{2} = ZAd; |
| let Inst{1-0} = imm; |
| } |
| |
| def _S : sme2_mova_vec_to_tile_vg2_multi_base<0b10, v, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| uimm1s2range, ZZ_s_mul_r, |
| mnemonic> { |
| bits<2> ZAd; |
| bits<1> imm; |
| let Inst{2-1} = ZAd; |
| let Inst{0} = imm; |
| } |
| |
| def _D : sme2_mova_vec_to_tile_vg2_multi_base<0b11, v, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| uimm0s2range, ZZ_d_mul_r, |
| mnemonic> { |
| bits<3> ZAd; |
| let Inst{2-0} = ZAd; |
| } |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm3s2range, ZZ_b_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm2s2range, ZZ_h_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm1s2range, ZZ_s_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s2range, ZZ_d_mul_r, |
| "mov">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm3s2range, ZZ_b_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm2s2range, ZZ_h_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm1s2range, ZZ_s_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s2range, ZZ_d_mul_r, |
| "mova">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm3s2range, ZZ_b_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm2s2range, ZZ_h_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm1s2range, ZZ_s_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s2range, ZZ_d_mul_r, |
| "mova">; |
| } |
| |
| multiclass sme2_mova_vec_to_tile_vg2_multi<string mnemonic>{ |
| defm _H : sme2_mova_vec_to_tile_vg2_multi_base<0b0, mnemonic>; |
| defm _V : sme2_mova_vec_to_tile_vg2_multi_base<0b1, mnemonic>; |
| } |
| |
| class sme2_mova_vec_to_tile_vg4_multi_base<bits<2> sz, bit v, bits<3> op, |
| RegisterOperand tile_ty, |
| Operand index_ty, |
| RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs tile_ty:$ZAd), |
| (ins tile_ty:$_ZAd, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm, |
| vector_ty:$Zn), |
| mnemonic, |
| "\t$ZAd[$Rs, $imm], $Zn", |
| "", []>, Sched<[]> { |
| bits<2> Rs; |
| bits<3> Zn; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-16} = 0b000100; |
| let Inst{15} = v; |
| let Inst{14-13} = Rs; |
| let Inst{12-10} = 0b001; |
| let Inst{9-7} = Zn; |
| let Inst{6-3} = 0b0000; |
| let Inst{2-0} = op; |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| // SME2 move vector to tile, four registers |
| multiclass sme2_mova_vec_to_tile_vg4_multi_base<bit v, string mnemonic> { |
| |
| def _B : sme2_mova_vec_to_tile_vg4_multi_base<0b00, v, {0,?,?}, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| uimm2s4range, ZZZZ_b_mul_r, |
| mnemonic> { |
| bits<2> imm; |
| let Inst{1-0} = imm; |
| } |
| |
| def _H : sme2_mova_vec_to_tile_vg4_multi_base<0b01, v, {0,?,?}, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| uimm1s4range, ZZZZ_h_mul_r, |
| mnemonic> { |
| bits<1> ZAd; |
| bits<1> imm; |
| let Inst{1} = ZAd; |
| let Inst{0} = imm; |
| } |
| |
| def _S : sme2_mova_vec_to_tile_vg4_multi_base<0b10, v, {0,?,?}, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| uimm0s4range, ZZZZ_s_mul_r, |
| mnemonic> { |
| bits<2> ZAd; |
| let Inst{1-0} = ZAd; |
| } |
| |
| def _D : sme2_mova_vec_to_tile_vg4_multi_base<0b11, v, {?,?,?}, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| uimm0s4range, ZZZZ_d_mul_r, |
| mnemonic> { |
| bits<3> ZAd; |
| let Inst{2-0} = ZAd; |
| } |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _B), |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm2s4range, ZZZZ_b_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _H), |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm1s4range, ZZZZ_h_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _S), |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, ZZZZ_s_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME # _D), |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, ZZZZ_d_mul_r, |
| "mov">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _B), |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm2s4range, ZZZZ_b_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _H), |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm1s4range, ZZZZ_h_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _S), |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, ZZZZ_s_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME # _D), |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, ZZZZ_d_mul_r, |
| "mova">; |
| |
| } |
| |
| multiclass sme2_mova_vec_to_tile_vg4_multi<string mnemonic>{ |
| defm _H : sme2_mova_vec_to_tile_vg4_multi_base<0b0, mnemonic>; |
| defm _V : sme2_mova_vec_to_tile_vg4_multi_base<0b1, mnemonic>; |
| } |
| |
| // SME Move into Array |
| class sme2_mova_vec_to_array_vg24_multi< bits<5> op, RegisterOperand array_ty, |
| RegisterOperand vector_ty, |
| string mnemonic, |
| string vg_acronym=""> |
| : I<(outs array_ty:$ZAd), |
| (ins array_ty:$_ZAd, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm, |
| vector_ty:$Zn), |
| mnemonic, "\t$ZAd[$Rs, $imm, " # vg_acronym # "], $Zn", |
| "", []>, Sched<[]> { |
| bits<2> Rs; |
| bits<3> imm; |
| let Inst{31-15} = 0b11000000000001000; |
| let Inst{14-13} = Rs; |
| let Inst{12-11} = 0b01; |
| let Inst{10-6} = op; |
| let Inst{5-3} = 0b000; |
| let Inst{2-0} = imm; |
| |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| // MOVA (vector to array, two registers) |
| multiclass sme2_mova_vec_to_array_vg2_multi<string mnemonic> { |
| def NAME : sme2_mova_vec_to_array_vg24_multi<{0,?,?,?,?}, MatrixOp64, |
| ZZ_d_mul_r, mnemonic, "vgx2">{ |
| bits<4> Zn; |
| let Inst{9-6} = Zn; |
| } |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_b_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_h_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_s_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_d_mul_r, |
| "mova">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_b_mul_r, |
| "mova", "vgx2">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_h_mul_r, |
| "mova", "vgx2">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_s_mul_r, |
| "mova", "vgx2">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_b_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_h_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_s_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_d_mul_r, |
| "mov">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_b_mul_r, |
| "mov", "vgx2">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_h_mul_r, |
| "mov", "vgx2">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_s_mul_r, |
| "mov", "vgx2">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZ_d_mul_r, |
| "mov", "vgx2">; |
| } |
| |
| // MOVA (vector to array, four registers) |
| multiclass sme2_mova_vec_to_array_vg4_multi<string mnemonic> { |
| def NAME : sme2_mova_vec_to_array_vg24_multi<{1,?,?,?,0}, MatrixOp64, |
| ZZZZ_d_mul_r, mnemonic, "vgx4"> { |
| bits<3> Zn; |
| let Inst{9-7} = Zn; |
| } |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_b_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_h_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_s_mul_r, |
| "mova">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_d_mul_r, |
| "mova">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_b_mul_r, |
| "mova", "vgx4">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_h_mul_r, |
| "mova", "vgx4">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_s_mul_r, |
| "mova", "vgx4">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_b_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_h_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_s_mul_r, |
| "mov">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_d_mul_r, |
| "mov">; |
| |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_b_mul_r, |
| "mov", "vgx4">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_h_mul_r, |
| "mov", "vgx4">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<0, !cast<Instruction>(NAME), |
| MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_s_mul_r, |
| "mov", "vgx4">; |
| defm : sme2_mova_vec_to_tile_or_array_aliases<1, !cast<Instruction>(NAME), |
| MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, ZZZZ_d_mul_r, |
| "mov", "vgx4">; |
| |
| } |
| |
| class sme2_mova_tile_to_vec_vg2_multi_base<bits<2> sz, bit v, bits<3> op, |
| RegisterOperand vector_ty, |
| RegisterOperand tile_ty, |
| Operand index_ty, |
| string mnemonic> |
| : I<!if(op{1}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), |
| (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), |
| mnemonic, |
| "\t$Zd, $ZAn[$Rs, $imm]", |
| "", []>, Sched<[]> { |
| bits<4> Zd; |
| bits<2> Rs; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-16} = 0b000110; |
| let Inst{15} = v; |
| let Inst{14-13} = Rs; |
| let Inst{12-11} = 0b00; |
| let Inst{10-8} = op; |
| let Inst{4-1} = Zd; |
| let Inst{0} = 0b0; |
| |
| let Constraints = !if(op{1}, "$ZAn = $_ZAn", ""); |
| } |
| |
| multiclass sme2_mova_tile_or_array_to_vec_aliases<int op, Instruction inst, |
| RegisterOperand vector_ty, |
| RegisterOperand tile_or_array_ty, |
| RegisterOperand rv_ty, |
| Operand index_ty, |
| string mnemonic, |
| string vg_acronym=""> { |
| def : InstAlias<mnemonic # "\t$Zd, $ZAn[$Rs, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", |
| (inst vector_ty:$Zd, tile_or_array_ty:$ZAn, rv_ty:$Rs, index_ty:$imm), op>; |
| |
| } |
| |
| multiclass sme2_mova_tile_to_vec_vg2_multi_inst<bit v, bits<3> opc, string mnemonic> { |
| |
| def _B : sme2_mova_tile_to_vec_vg2_multi_base<0b00, v, opc, ZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| uimm3s2range, mnemonic> { |
| bits<3> imm; |
| let Inst{7-5} = imm; |
| } |
| |
| def _H : sme2_mova_tile_to_vec_vg2_multi_base<0b01, v, opc, ZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| uimm2s2range, mnemonic> { |
| bits<1> ZAn; |
| bits<2> imm; |
| let Inst{7} = ZAn; |
| let Inst{6-5} = imm; |
| } |
| |
| def _S : sme2_mova_tile_to_vec_vg2_multi_base<0b10, v, opc, ZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| uimm1s2range, mnemonic> { |
| bits<2> ZAn; |
| bits<1> imm; |
| let Inst{7-6} = ZAn; |
| let Inst{5} = imm; |
| } |
| |
| def _D : sme2_mova_tile_to_vec_vg2_multi_base<0b11, v, opc, ZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| uimm0s2range, mnemonic> { |
| bits<3> ZAn; |
| let Inst{7-5} = ZAn; |
| } |
| |
| if !eq(mnemonic, "mova") then { |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _B), |
| ZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm3s2range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1,!cast<Instruction>(NAME # _H), |
| ZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm2s2range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), |
| ZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm1s2range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), |
| ZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s2range, "mov">; |
| } |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _B), |
| ZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm3s2range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0,!cast<Instruction>(NAME # _H), |
| ZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm2s2range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), |
| ZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm1s2range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), |
| ZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s2range, mnemonic>; |
| |
| } |
| |
| // SME2 move tile to vector, two registers |
| multiclass sme2_mova_tile_to_vec_vg2_multi<string mnemonic>{ |
| defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b000, mnemonic>; |
| defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b000, mnemonic>; |
| } |
| |
| // SME2p1 move tile to vector and zero tile, two registers |
| multiclass sme2p1_movaz_tile_to_vec_vg2<string mnemonic>{ |
| defm _H : sme2_mova_tile_to_vec_vg2_multi_inst<0b0, 0b010, mnemonic>; |
| defm _V : sme2_mova_tile_to_vec_vg2_multi_inst<0b1, 0b010, mnemonic>; |
| } |
| |
| class sme2_mova_tile_to_vec_vg4_multi_base<bits<2> sz, bit v, bits<6> op, |
| RegisterOperand vector_ty, |
| RegisterOperand tile_ty, |
| Operand index_ty, |
| string mnemonic> |
| : I<!if(op{4}, (outs vector_ty:$Zd, tile_ty:$_ZAn), (outs vector_ty:$Zd)), |
| (ins tile_ty:$ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), |
| mnemonic, |
| "\t$Zd, $ZAn[$Rs, $imm]", |
| "", []>, Sched<[]> { |
| bits<3> Zd; |
| bits<2> Rs; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-16} = 0b000110; |
| let Inst{15} = v; |
| let Inst{14-13} = Rs; |
| let Inst{12-11} = 0b00; |
| let Inst{10-5} = op{5-0}; |
| let Inst{4-2} = Zd; |
| let Inst{1-0} = 0b00; |
| |
| let Constraints = !if(op{4}, "$ZAn = $_ZAn", ""); |
| } |
| |
| multiclass sme2_mova_tile_to_vec_vg4_multi_base<bit v, bits<3> opc, string mnemonic> { |
| |
| def _B : sme2_mova_tile_to_vec_vg4_multi_base<0b00, v, {opc,0,?,?}, |
| ZZZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| uimm2s4range, mnemonic> { |
| bits<2> imm; |
| let Inst{6-5} = imm; |
| } |
| |
| def _H : sme2_mova_tile_to_vec_vg4_multi_base<0b01, v, {opc,0,?,?}, |
| ZZZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| uimm1s4range, mnemonic> { |
| bits<1> ZAn; |
| bits<1> imm; |
| let Inst{6} = ZAn; |
| let Inst{5} = imm; |
| } |
| |
| def _S : sme2_mova_tile_to_vec_vg4_multi_base<0b10, v, {opc,0,?,?}, |
| ZZZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| uimm0s4range, mnemonic> { |
| bits<2> ZAn; |
| let Inst{6-5} = ZAn; |
| } |
| |
| def _D : sme2_mova_tile_to_vec_vg4_multi_base<0b11, v, {opc,?,?,?}, |
| ZZZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| uimm0s4range, mnemonic> { |
| bits<3> ZAn; |
| let Inst{7-5} = ZAn; |
| } |
| |
| if !eq(mnemonic, "mova") then { |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _B), |
| ZZZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm2s4range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _H), |
| ZZZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm1s4range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _S), |
| ZZZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME # _D), |
| ZZZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, "mov">; |
| } |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _B), |
| ZZZZ_b_mul_r, |
| !if(v, TileVectorOpV8, |
| TileVectorOpH8), |
| MatrixIndexGPR32Op12_15, |
| uimm2s4range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _H), |
| ZZZZ_h_mul_r, |
| !if(v, TileVectorOpV16, |
| TileVectorOpH16), |
| MatrixIndexGPR32Op12_15, |
| uimm1s4range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _S), |
| ZZZZ_s_mul_r, |
| !if(v, TileVectorOpV32, |
| TileVectorOpH32), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME # _D), |
| ZZZZ_d_mul_r, |
| !if(v, TileVectorOpV64, |
| TileVectorOpH64), |
| MatrixIndexGPR32Op12_15, |
| uimm0s4range, mnemonic>; |
| |
| } |
| |
| // SME2 move tile to vector, four registers |
| multiclass sme2_mova_tile_to_vec_vg4_multi<string mnemonic>{ |
| defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b100, mnemonic>; |
| defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b100, mnemonic>; |
| } |
| |
| // SME2p1 move tile to vector and zero tile, four registers |
| multiclass sme2p1_movaz_tile_to_vec_vg4<string mnemonic>{ |
| defm _H : sme2_mova_tile_to_vec_vg4_multi_base<0b0, 0b110, mnemonic>; |
| defm _V : sme2_mova_tile_to_vec_vg4_multi_base<0b1, 0b110, mnemonic>; |
| } |
| |
| |
| class sme2_mova_array_to_vec_vg24_multi<bits<4>op, RegisterOperand vector_ty, |
| RegisterOperand array_ty, |
| string mnemonic, string vg_acronym> |
| : I<!if(op{2}, (outs vector_ty:$Zd, array_ty:$_ZAn), (outs vector_ty:$Zd)), |
| (ins array_ty:$ZAn, MatrixIndexGPR32Op8_11:$Rs, sme_elm_idx0_7:$imm), |
| mnemonic, |
| "\t$Zd, $ZAn[$Rs, $imm, " # vg_acronym # "]", |
| "", []>, Sched<[]> { |
| bits<2> Rs; |
| bits<3> imm; |
| let Inst{31-15} = 0b11000000000001100; |
| let Inst{14-13} = Rs; |
| let Inst{12-11} = 0b01; |
| let Inst{10-8} = op{3-1}; |
| let Inst{7-5} = imm; |
| let Inst{1} = op{0}; |
| let Inst{0} = 0b0; |
| let Constraints = !if(op{2}, "$ZAn = $_ZAn", ""); |
| } |
| |
| // move array to vector, two registers. |
| multiclass sme2_mova_array_to_vec_vg2_multi<bits<3> opc, string mnemonic> { |
| def NAME : sme2_mova_array_to_vec_vg24_multi<{opc,?}, ZZ_d_mul_r, MatrixOp64, |
| mnemonic, "vgx2"> { |
| bits<4> Zd; |
| let Inst{4-1} = Zd; |
| } |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx2">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx2">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx2">; |
| |
| if !eq(mnemonic, "mova") then { |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx2">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx2">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx2">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), |
| ZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx2">; |
| } |
| } |
| |
| // move array to vector, four registers |
| multiclass sme2_mova_array_to_vec_vg4_multi<bits<4> opc, string mnemonic> { |
| def NAME : sme2_mova_array_to_vec_vg24_multi<opc, ZZZZ_d_mul_r, MatrixOp64, |
| mnemonic, "vgx4"> { |
| bits<3> Zd; |
| let Inst{4-2} = Zd; |
| } |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic>; |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx4">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx4">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, mnemonic, "vgx4">; |
| |
| if !eq(mnemonic, "mova") then { |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov">; |
| |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_b_mul_r, MatrixOp8, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx4">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_h_mul_r, MatrixOp16, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx4">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<0, !cast<Instruction>(NAME), |
| ZZZZ_s_mul_r, MatrixOp32, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx4">; |
| defm : sme2_mova_tile_or_array_to_vec_aliases<1, !cast<Instruction>(NAME), |
| ZZZZ_d_mul_r, MatrixOp64, |
| MatrixIndexGPR32Op8_11, |
| sme_elm_idx0_7, "mov", "vgx4">; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 multi-vec saturating shift right narrow |
| class sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> |
| : I<(outs ZPR16:$Zd), (ins ZZ_s_mul_r:$Zn, vecshiftR16:$imm4), |
| mnemonic, "\t$Zd, $Zn, $imm4", |
| "", []>, Sched<[]> { |
| bits<4> imm4; |
| bits<4> Zn; |
| bits<5> Zd; |
| let Inst{31-21} = 0b11000001111; |
| let Inst{20} = op; |
| let Inst{19-16} = imm4; |
| let Inst{15-10} = 0b110101; |
| let Inst{9-6} = Zn; |
| let Inst{5} = u; |
| let Inst{4-0} = Zd; |
| } |
| |
| multiclass sme2_sat_shift_vector_vg2<string mnemonic, bit op, bit u> { |
| def _H : sme2_sat_shift_vector_vg2<mnemonic, op, u>; |
| } |
| |
| class sme2_sat_shift_vector_vg4<bits<2> sz, bits<3> op, ZPRRegOp zpr_ty, |
| RegisterOperand vector_ty, Operand imm_ty, |
| string mnemonic> |
| : I<(outs zpr_ty:$Zd), (ins vector_ty:$Zn, imm_ty:$imm), |
| mnemonic, "\t$Zd, $Zn, $imm", |
| "", []>, Sched<[]> { |
| bits<3> Zn; |
| bits<5> Zd; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| // Inst{20-16} = imm5; |
| let Inst{15-11} = 0b11011; |
| let Inst{10} = op{2}; |
| let Inst{9-7} = Zn; |
| let Inst{6-5} = op{1-0}; |
| let Inst{4-0} = Zd; |
| } |
| |
| multiclass sme2_sat_shift_vector_vg4<string mnemonic, bits<3> op> { |
| def _B : sme2_sat_shift_vector_vg4<{0,1}, op, ZPR8, ZZZZ_s_mul_r, vecshiftR32, |
| mnemonic>{ |
| bits<5> imm; |
| let Inst{20-16} = imm; |
| } |
| def _H : sme2_sat_shift_vector_vg4<{1,?}, op, ZPR16, ZZZZ_d_mul_r, vecshiftR64, |
| mnemonic> { |
| bits<6> imm; |
| let Inst{22} = imm{5}; |
| let Inst{20-16} = imm{4-0}; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Multi-vector - SVE Select |
| class sme2_sel_vector_vg24<bits<2> sz, bits<4> op, RegisterOperand vector_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zd), |
| (ins PNRAny_p8to15:$PNg, vector_ty:$Zn, vector_ty:$Zm), |
| mnemonic, "\t$Zd, $PNg, $Zn, $Zm", |
| "", []>, Sched<[]> { |
| bits<3> PNg; |
| let Inst{31-24} = 0b11000001; |
| let Inst{23-22} = sz; |
| let Inst{21} = 0b1; |
| let Inst{17-16} = op{3-2}; |
| let Inst{15-13} = 0b100; |
| let Inst{12-10} = PNg; |
| let Inst{6} = op{1}; |
| let Inst{5} = 0b0; |
| let Inst{1} = op{0}; |
| let Inst{0} = 0b0; |
| } |
| |
| class sme2_sel_vector_vg2<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_sel_vector_vg24<sz, {?,0,?,?}, vector_ty, mnemonic> { |
| bits<4> Zm; |
| bits<4> Zn; |
| bits<4> Zd; |
| let Inst{20-17} = Zm; |
| let Inst{9-6} = Zn; |
| let Inst{4-1} = Zd; |
| } |
| |
| multiclass sme2_sel_vector_vg2<string mnemonic>{ |
| def _B : sme2_sel_vector_vg2<0b00, ZZ_b_mul_r, mnemonic>; |
| def _H : sme2_sel_vector_vg2<0b01, ZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sel_vector_vg2<0b10, ZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sel_vector_vg2<0b11, ZZ_d_mul_r, mnemonic>; |
| } |
| class sme2_sel_vector_vg4<bits<2> sz, RegisterOperand vector_ty, |
| string mnemonic> |
| : sme2_sel_vector_vg24<sz, 0b0100, vector_ty, mnemonic> { |
| bits<3> Zm; |
| bits<3> Zn; |
| bits<3> Zd; |
| let Inst{20-18} = Zm; |
| let Inst{9-7} = Zn; |
| let Inst{4-2} = Zd; |
| } |
| multiclass sme2_sel_vector_vg4<string mnemonic> { |
| def _B : sme2_sel_vector_vg4<0b00, ZZZZ_b_mul_r, mnemonic>; |
| def _H : sme2_sel_vector_vg4<0b01, ZZZZ_h_mul_r, mnemonic>; |
| def _S : sme2_sel_vector_vg4<0b10, ZZZZ_s_mul_r, mnemonic>; |
| def _D : sme2_sel_vector_vg4<0b11, ZZZZ_d_mul_r, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Non contiguous Load and Store |
| |
| class sme2_ld_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| RegisterOperand gpr_ty, |
| string mnemonic> |
| : I<(outs multi_vector_ty:$Zt), |
| (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), |
| mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", |
| "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<3> PNg; |
| bits<5> Rn; |
| bits<4> Zt; |
| let Inst{31-21} = 0b10100001000; |
| let Inst{20-16} = Rm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = Zt{3}; |
| let Inst{3} = n; |
| let Inst{2-0} = Zt{2-0}; |
| |
| let mayLoad = 1; |
| } |
| |
| class sme2_ld_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| RegisterOperand gpr_ty, |
| string mnemonic> |
| : I<(outs multi_vector_ty:$Zt), |
| (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), |
| mnemonic, "\t$Zt, $PNg/z, [$Rn, $Rm]", |
| "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<3> PNg; |
| bits<5> Rn; |
| bits<3> Zt; |
| let Inst{31-21} = 0b10100001000; |
| let Inst{20-16} = Rm; |
| let Inst{15} = 0b1; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = Zt{2}; |
| let Inst{3} = n; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = Zt{1-0}; |
| |
| let mayLoad = 1; |
| } |
| |
| class sme2_ld_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic> |
| : I<(outs multi_vector_ty:$Zt), |
| (ins PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), |
| mnemonic, "\t$Zt, $PNg/z, [$Rn, $imm4, mul vl]", |
| "", []>, Sched<[]> { |
| bits<4> imm4; |
| bits<3> PNg; |
| bits<5> Rn; |
| let Inst{31-20} = 0b101000010100; |
| let Inst{19-16} = imm4; |
| let Inst{15} = op{1}; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{3} = n; |
| let Inst{2} = op{0}; |
| |
| let mayLoad = 1; |
| } |
| |
| multiclass sme2_ld_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic>{ |
| def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, |
| multi_vector_ty, |
| index_ty, mnemonic> { |
| bits<4> Zt; |
| let Inst{4} = Zt{3}; |
| let Inst{2-0} = Zt{2-0}; |
| } |
| |
| def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", |
| (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; |
| } |
| |
| multiclass sme2_ld_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic> { |
| def NAME : sme2_ld_vector_vg24_multi_scalar_immediate<msz, n, 0b10, |
| multi_vector_ty, |
| index_ty, mnemonic> { |
| bits<3> Zt; |
| let Inst{4} = Zt{2}; |
| let Inst{1-0} = Zt{1-0}; |
| } |
| |
| def : InstAlias<mnemonic # "\t$Zt, $PNg/z, [$Rn]", |
| (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, 0), 1>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2 Non-Contiguous Store |
| class sme2_st_vector_vg2_multi_scalar_scalar<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| RegisterOperand gpr_ty, |
| string mnemonic> |
| : I<(outs ), |
| (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), |
| mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", |
| "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<3> PNg; |
| bits<5> Rn; |
| bits<4> Zt; |
| let Inst{31-21} = 0b10100001001; |
| let Inst{20-16} = Rm; |
| let Inst{15} = 0b0; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = Zt{3}; |
| let Inst{3} = n; |
| let Inst{2-0} = Zt{2-0}; |
| |
| let mayStore = 1; |
| } |
| |
| class sme2_st_vector_vg4_multi_scalar_scalar<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| RegisterOperand gpr_ty, |
| string mnemonic> |
| : I<(outs ), |
| (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, gpr_ty:$Rm), |
| mnemonic, "\t$Zt, $PNg, [$Rn, $Rm]", |
| "", []>, Sched<[]> { |
| bits<5> Rm; |
| bits<3> PNg; |
| bits<5> Rn; |
| bits<3> Zt; |
| let Inst{31-21} = 0b10100001001; |
| let Inst{20-16} = Rm; |
| let Inst{15} = 0b1; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{4} = Zt{2}; |
| let Inst{3} = n; |
| let Inst{2} = 0b0; |
| let Inst{1-0} = Zt{1-0}; |
| |
| let mayStore = 1; |
| } |
| |
| class sme2_st_vector_vg24_multi_scalar_immediate<bits<2> msz, bit n, bits<2> op, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic> |
| : I<(outs ), |
| (ins multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn, index_ty:$imm4), |
| mnemonic, "\t$Zt, $PNg, [$Rn, $imm4, mul vl]", |
| "", []>, Sched<[]> { |
| bits<4> imm4; |
| bits<3> PNg; |
| bits<5> Rn; |
| let Inst{31-20} = 0b101000010110; |
| let Inst{19-16} = imm4; |
| let Inst{15} = op{1}; |
| let Inst{14-13} = msz; |
| let Inst{12-10} = PNg; |
| let Inst{9-5} = Rn; |
| let Inst{3} = n; |
| let Inst{2} = op{0}; |
| |
| let mayStore = 1; |
| } |
| |
| |
| multiclass sme2_st_vector_vg2_multi_scalar_immediate<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic> { |
| def NAME: sme2_st_vector_vg24_multi_scalar_immediate<msz, n, {0,?}, |
| multi_vector_ty, |
| index_ty, mnemonic> { |
| bits<4> Zt; |
| let Inst{4} = Zt{3}; |
| let Inst{2-0} = Zt{2-0}; |
| } |
| |
| def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", |
| (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; |
| } |
| |
| multiclass sme2_st_vector_vg4_multi_scalar_immediate<bits<2> msz, bit n, |
| RegisterOperand multi_vector_ty, |
| Operand index_ty, |
| string mnemonic> { |
| def NAME : sme2_st_vector_vg24_multi_scalar_immediate<msz, n, 0b10, |
| multi_vector_ty, |
| index_ty, mnemonic> { |
| bits<3> Zt; |
| let Inst{4} = Zt{2}; |
| let Inst{1-0} = Zt{1-0}; |
| } |
| |
| def : InstAlias<mnemonic # "\t$Zt, $PNg, [$Rn]", |
| (!cast<Instruction>(NAME) multi_vector_ty:$Zt, PNRAny_p8to15:$PNg, GPR64sp:$Rn,0), 1>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2.1 |
| //===----------------------------------------------------------------------===// |
| // SME zeroing move array to vector |
| class sme2p1_movaz_tile_to_vec_base<bits<2> sz, bit q, bit v, ZPRRegOp vector_ty, |
| RegisterOperand tile_ty, Operand index_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zd, tile_ty:$ZAn), |
| (ins tile_ty:$_ZAn, MatrixIndexGPR32Op12_15:$Rs, index_ty:$imm), |
| mnemonic, "\t$Zd, $ZAn[$Rs, $imm]", |
| "", []>, Sched<[]> { |
| bits<2> Rs; |
| bits<5> Zd; |
| let Inst{31-24} = 0b11000000; |
| let Inst{23-22} = sz; |
| let Inst{21-17} = 0b00001; |
| let Inst{16} = q; |
| let Inst{15} = v; |
| let Inst{14-13} = Rs; |
| let Inst{12-9} = 0b0001; |
| let Inst{4-0} = Zd; |
| let Constraints = "$ZAn = $_ZAn"; |
| } |
| |
| multiclass sme2p1_movaz_tile_to_vec_base<bit v, string mnemonic> { |
| def _B : sme2p1_movaz_tile_to_vec_base<0b00, 0b0, v, ZPR8, |
| !if(v, TileVectorOpV8, TileVectorOpH8), |
| sme_elm_idx0_15, mnemonic> { |
| bits<4> imm; |
| let Inst{8-5} = imm; |
| } |
| |
| def _H : sme2p1_movaz_tile_to_vec_base<0b01, 0b0, v, ZPR16, |
| !if(v, TileVectorOpV16, TileVectorOpH16), |
| sme_elm_idx0_7, mnemonic> { |
| bits<1> ZAn; |
| bits<3> imm; |
| let Inst{8} = ZAn; |
| let Inst{7-5} = imm; |
| } |
| |
| def _S : sme2p1_movaz_tile_to_vec_base<0b10, 0b0, v, ZPR32, |
| !if(v, TileVectorOpV32, TileVectorOpH32), |
| sme_elm_idx0_3, mnemonic> { |
| bits<2> ZAn; |
| bits<2> imm; |
| let Inst{8-7} = ZAn; |
| let Inst{6-5} = imm; |
| } |
| |
| def _D : sme2p1_movaz_tile_to_vec_base<0b11, 0b0, v, ZPR64, |
| !if(v, TileVectorOpV64, TileVectorOpH64), |
| sme_elm_idx0_1, mnemonic> { |
| bits<3> ZAn; |
| bits<1> imm; |
| let Inst{8-6} = ZAn; |
| let Inst{5} = imm; |
| } |
| |
| def _Q : sme2p1_movaz_tile_to_vec_base<0b11, 0b1, v, ZPR128, |
| !if(v, TileVectorOpV128, TileVectorOpH128), |
| sme_elm_idx0_0, mnemonic> { |
| bits<4> ZAn; |
| let Inst{8-5} = ZAn; |
| } |
| } |
| |
| multiclass sme2p1_movaz_tile_to_vec<string mnemonic>{ |
| defm _H : sme2p1_movaz_tile_to_vec_base<0b0, mnemonic>; |
| defm _V : sme2p1_movaz_tile_to_vec_base<0b1, mnemonic>; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2.1 multiple vectors zero array |
| |
| class sme2p1_zero_matrix<bits<6> opc, Operand index_ty, string mnemonic, |
| string vg_acronym=""> |
| : I<(outs MatrixOp64:$ZAd), |
| (ins MatrixOp64:$_ZAd, MatrixIndexGPR32Op8_11:$Rv, index_ty:$imm), |
| mnemonic, "\t$ZAd[$Rv, $imm" # !if(!eq(vg_acronym, ""), "", ", " # vg_acronym) # "]", |
| "", []>, Sched<[]> { |
| bits <2> Rv; |
| let Inst{31-18} = 0b11000000000011; |
| let Inst{17-15} = opc{5-3}; |
| let Inst{14-13} = Rv; |
| let Inst{12-3} = 0b0000000000; |
| let Inst{2-0} = opc{2-0}; |
| let Constraints = "$ZAd = $_ZAd"; |
| } |
| |
| multiclass sme2p1_zero_matrix<string mnemonic> { |
| def _VG2_Z : sme2p1_zero_matrix<{0b000,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx2"> { |
| bits<3> imm; |
| let Inst{2-0} = imm; |
| } |
| def _2Z : sme2p1_zero_matrix<{0b001,?,?,?}, uimm3s2range, mnemonic> { |
| bits<3> imm; |
| let Inst{2-0} = imm; |
| } |
| def _VG2_2Z : sme2p1_zero_matrix<{0b0100,?,?}, uimm2s2range, mnemonic, "vgx2"> { |
| bits<2> imm; |
| let Inst{1-0} = imm; |
| } |
| def _VG4_2Z : sme2p1_zero_matrix<{0b0110,?,?}, uimm2s2range, mnemonic, "vgx4"> { |
| bits<2> imm; |
| let Inst{1-0} = imm; |
| } |
| def _VG4_Z : sme2p1_zero_matrix<{0b100,?,?,?}, sme_elm_idx0_7, mnemonic, "vgx4"> { |
| bits<3> imm; |
| let Inst{2-0} = imm; |
| } |
| def _4Z : sme2p1_zero_matrix<{0b1010,?,?}, uimm2s4range, mnemonic> { |
| bits<2> imm; |
| let Inst{1-0} = imm; |
| } |
| def _VG2_4Z :sme2p1_zero_matrix<{0b11000,?}, uimm1s4range, mnemonic, "vgx2"> { |
| bits<1> imm; |
| let Inst{0} = imm; |
| } |
| def _VG4_4Z :sme2p1_zero_matrix<{0b11100,?}, uimm1s4range, mnemonic, "vgx4"> { |
| bits<1> imm; |
| let Inst{0} = imm; |
| } |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // SME2.1 lookup table expand two non-contiguous registers |
| |
| class sme2p1_luti_vector_vg2_index<bits<4> op, bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), |
| mnemonic, "\t$Zd, $ZTt, $Zn$i", |
| "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<4> Zd; |
| let Inst{31-19} = 0b1100000010011; |
| let Inst{18-15} = op; |
| let Inst{14} = 0b1; |
| let Inst{13-12} = sz; |
| let Inst{11-10} = 0b00; |
| let Inst{9-5} = Zn; |
| let Inst{4} = Zd{3}; |
| let Inst{3} = 0b0; |
| let Inst{2-0} = Zd{2-0}; |
| } |
| |
| class sme2p1_luti2_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : sme2p1_luti_vector_vg2_index<{1,?,?,?}, sz, vector_ty, index_ty, mnemonic> { |
| bits<3> i; |
| let Inst{17-15} = i; |
| } |
| |
| multiclass sme2p1_luti2_vector_vg2_index<string mnemonic> { |
| def _B : sme2p1_luti2_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexH, |
| mnemonic>; |
| def _H : sme2p1_luti2_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexH, |
| mnemonic>; |
| } |
| |
| class sme2p1_luti4_vector_vg2_index<bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : sme2p1_luti_vector_vg2_index<{0b01,?,?}, sz, vector_ty, index_ty, mnemonic> { |
| bits<2> i; |
| let Inst{16-15} = i; |
| } |
| multiclass sme2p1_luti4_vector_vg2_index<string mnemonic> { |
| def _B : sme2p1_luti4_vector_vg2_index<0b00, ZZ_b_strided, VectorIndexS, |
| mnemonic>; |
| def _H : sme2p1_luti4_vector_vg2_index<0b01, ZZ_h_strided, VectorIndexS, |
| mnemonic>; |
| } |
| |
| // SME2.1 lookup table expand four non-contiguous registers |
| class sme2p1_luti_vector_vg4_index<bits<3> op, bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : I<(outs vector_ty:$Zd), (ins ZTR:$ZTt, ZPRAny:$Zn, index_ty:$i), |
| mnemonic, "\t$Zd, $ZTt, $Zn$i", |
| "", []>, Sched<[]> { |
| bits<5> Zn; |
| bits<3> Zd; |
| let Inst{31-19} = 0b1100000010011; |
| let Inst{18-16} = op; |
| let Inst{15-14} = 0b10; |
| let Inst{13-12} = sz; |
| let Inst{11-10} = 0b00; |
| let Inst{9-5} = Zn; |
| let Inst{4} = Zd{2}; |
| let Inst{3-2} = 0b00; |
| let Inst{1-0} = Zd{1-0}; |
| } |
| |
| class sme2p1_luti2_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : sme2p1_luti_vector_vg4_index<{1,?,?}, sz, vector_ty, index_ty, mnemonic> { |
| bits<2> i; |
| let Inst{17-16} = i; |
| } |
| |
| multiclass sme2p1_luti2_vector_vg4_index<string mnemonic> { |
| def _B : sme2p1_luti2_vector_vg4_index<0b00, ZZZZ_b_strided, VectorIndexS, |
| mnemonic>; |
| def _H : sme2p1_luti2_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexS, |
| mnemonic>; |
| } |
| |
| class sme2p1_luti4_vector_vg4_index<bits<2> sz, RegisterOperand vector_ty, |
| AsmVectorIndexOpnd index_ty, |
| string mnemonic> |
| : sme2p1_luti_vector_vg4_index<{0b01,?}, sz, vector_ty, index_ty, mnemonic> { |
| bit i; |
| let Inst{16} = i; |
| } |
| |
| multiclass sme2p1_luti4_vector_vg4_index<string mnemonic> { |
| def _H: sme2p1_luti4_vector_vg4_index<0b01, ZZZZ_h_strided, VectorIndexD, mnemonic>; |
| } |
