| //===----------- VVPInstrPatternsVec.td - VVP_* SDNode patterns -----------===// | |
| // | |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| // | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // This file describes how VVP_* SDNodes are lowered to machine instructions. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| //===----------------------------------------------------------------------===// | |
| // | |
| // VVP SDNode definitions. | |
| // | |
| //===----------------------------------------------------------------------===// | |
| include "VVPInstrInfo.td" | |
| multiclass VectorStore<ValueType DataVT, | |
| ValueType PtrVT, ValueType MaskVT, | |
| string STWithMask, string STNoMask> { | |
| // Unmasked (imm stride). | |
| def : Pat<(vvp_store | |
| DataVT:$val, PtrVT:$addr, | |
| (i64 simm7:$stride), (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(STNoMask#"irvl") | |
| (LO7 $stride), $addr, $val, $avl)>; | |
| // Unmasked. | |
| def : Pat<(vvp_store | |
| DataVT:$val, PtrVT:$addr, | |
| i64:$stride, (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(STNoMask#"rrvl") | |
| $stride, $addr, $val, $avl)>; | |
| // Masked (imm stride). | |
| def : Pat<(vvp_store | |
| DataVT:$val, PtrVT:$addr, | |
| (i64 simm7:$stride), MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(STWithMask#"irvml") | |
| (LO7 $stride), $addr, $val, $mask, $avl)>; | |
| // Masked. | |
| def : Pat<(vvp_store | |
| DataVT:$val, PtrVT:$addr, | |
| i64:$stride, MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(STWithMask#"rrvml") | |
| $stride, $addr, $val, $mask, $avl)>; | |
| } | |
| defm : VectorStore<v256f64, i64, v256i1, "VST", "VST">; | |
| defm : VectorStore<v256i64, i64, v256i1, "VST", "VST">; | |
| defm : VectorStore<v256f32, i64, v256i1, "VSTU", "VSTU">; | |
| defm : VectorStore<v256i32, i64, v256i1, "VSTL", "VSTL">; | |
| multiclass VectorLoad<ValueType DataVT, | |
| ValueType PtrVT, ValueType MaskVT, | |
| string GTWithMask, string LDNoMask> { | |
| // Unmasked (imm stride). | |
| def : Pat<(DataVT (vvp_load | |
| PtrVT:$addr, (i64 simm7:$stride), | |
| (MaskVT true_mask), i32:$avl)), | |
| (!cast<Instruction>(LDNoMask#"irl") | |
| (LO7 $stride), $addr, $avl)>; | |
| // Unmasked. | |
| def : Pat<(DataVT (vvp_load | |
| PtrVT:$addr, i64:$stride, | |
| (MaskVT true_mask), i32:$avl)), | |
| (!cast<Instruction>(LDNoMask#"rrl") | |
| $stride, PtrVT:$addr, $avl)>; | |
| // Masked (imm stride). | |
| def : Pat<(DataVT (vvp_load | |
| PtrVT:$addr, (i64 simm7:$stride), | |
| MaskVT:$mask, i32:$avl)), | |
| (!cast<Instruction>(GTWithMask#"vizml") | |
| (VADDULrvml $addr, | |
| (VMULULivml (LO7 $stride), (VSEQl $avl), $mask, $avl), | |
| $mask, $avl), | |
| 0, 0, | |
| $mask, | |
| $avl)>; | |
| // Masked. | |
| def : Pat<(DataVT (vvp_load | |
| PtrVT:$addr, i64:$stride, MaskVT:$mask, i32:$avl)), | |
| (!cast<Instruction>(GTWithMask#"vizml") | |
| (VADDULrvml $addr, | |
| (VMULULrvml $stride, (VSEQl $avl), $mask, $avl), | |
| $mask, $avl), | |
| 0, 0, | |
| $mask, | |
| $avl)>; | |
| } | |
| defm : VectorLoad<v256f64, i64, v256i1, "VGT", "VLD">; | |
| defm : VectorLoad<v256i64, i64, v256i1, "VGT", "VLD">; | |
| defm : VectorLoad<v256f32, i64, v256i1, "VGTU", "VLDU">; | |
| defm : VectorLoad<v256i32, i64, v256i1, "VGTLZX", "VLDLZX">; | |
| // Vector Gather and scatter | |
| multiclass VectorGather<ValueType DataVT, | |
| ValueType PtrVT, ValueType MaskVT, | |
| string GTPrefix> { | |
| // Unmasked. | |
| def : Pat<(DataVT (vvp_gather | |
| PtrVT:$addr, (MaskVT true_mask), i32:$avl)), | |
| (!cast<Instruction>(GTPrefix#"vizl") $addr, 0, 0, $avl)>; | |
| // Masked. | |
| def : Pat<(DataVT (vvp_gather PtrVT:$addr, MaskVT:$mask, i32:$avl)), | |
| (!cast<Instruction>(GTPrefix#"vizml") $addr, 0, 0, $mask, $avl)>; | |
| } | |
| defm : VectorGather<v256f64, v256i64, v256i1, "VGT">; | |
| defm : VectorGather<v256i64, v256i64, v256i1, "VGT">; | |
| defm : VectorGather<v256f32, v256i64, v256i1, "VGTU">; | |
| defm : VectorGather<v256i32, v256i64, v256i1, "VGTLZX">; | |
| multiclass VectorScatter<ValueType DataVT, | |
| ValueType PtrVT, ValueType MaskVT, | |
| string SCPrefix> { | |
| // Unmasked. | |
| def : Pat<(vvp_scatter | |
| DataVT:$data, PtrVT:$addr, (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(SCPrefix#"vizvl") $addr, 0, 0, $data, $avl)>; | |
| // Masked. | |
| def : Pat<(vvp_scatter | |
| DataVT:$data, PtrVT:$addr, MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(SCPrefix#"vizvml") $addr, 0, 0, $data, $mask, $avl)>; | |
| } | |
| defm : VectorScatter<v256f64, v256i64, v256i1, "VSC">; | |
| defm : VectorScatter<v256i64, v256i64, v256i1, "VSC">; | |
| defm : VectorScatter<v256f32, v256i64, v256i1, "VSCU">; | |
| defm : VectorScatter<v256i32, v256i64, v256i1, "VSCL">; | |
| /// FNEG { | |
| // Directly modify the sign bit to flip the sign. | |
| // Set sign bits in a pack of <2 x f32>. | |
| def packed_fneg_imm : OutPatFrag<(ins ), | |
| (i64 (SLLri (i64 (ORim 1, (i32 32))), 31))>; | |
| multiclass FNeg<ValueType DataVT> { | |
| // Masked with select. | |
| def : Pat<(vvp_select (vvp_fneg DataVT:$vx, (v256i1 srcvalue), (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| v256i1:$mask, | |
| i32:$avl), | |
| (VXORmvml_v (i32 1), $vx, $mask, $avl, $vfalse)>; | |
| // Unmasked. | |
| def : Pat<(vvp_fneg DataVT:$vx, (v256i1 true_mask), i32:$avl), | |
| (VXORmvl (i32 1), $vx, $avl)>; | |
| // Masked. | |
| def : Pat<(vvp_fneg DataVT:$vx, v256i1:$mask, i32:$avl), | |
| (VXORmvml (i32 1), $vx, $mask, $avl)>; | |
| } | |
| defm: FNeg<v256f32>; | |
| defm: FNeg<v256f64>; | |
| ///// Packed FNeg ///// | |
| // Masked with select. | |
| def : Pat<(vvp_select (vvp_fneg v512f32:$vx, (v512i1 srcvalue), (i32 srcvalue)), | |
| v512f32:$vfalse, | |
| v512i1:$mask, | |
| i32:$avl), | |
| (v512f32 (PVXORrvml_v (packed_fneg_imm ), $vx, $mask, $avl, $vfalse))>; | |
| // Unmasked. | |
| def : Pat<(vvp_fneg v512f32:$vx, (v512i1 true_mask), i32:$avl), | |
| (v512f32 (PVXORrvl (packed_fneg_imm ), $vx, $avl))>; | |
| // Masked. | |
| def : Pat<(vvp_fneg v512f32:$vx, v512i1:$mask, i32:$avl), | |
| (v512f32 (PVXORrvml (packed_fneg_imm ), $vx, $mask, $avl))>; | |
| /// } FNEG | |
| multiclass Binary_rv<SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru, broadcast. | |
| def : Pat<(vvp_select | |
| (OpNode | |
| (any_broadcast ScalarVT:$sx), | |
| DataVT:$vy, | |
| (MaskVT srcvalue), | |
| (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$pivot), | |
| (!cast<Instruction>(OpBaseName#"rvml_v") | |
| ScalarVT:$sx, | |
| $vy, | |
| $mask, | |
| $pivot, | |
| $vfalse)>; | |
| // Unmasked, broadcast. | |
| def : Pat<(OpNode | |
| (any_broadcast ScalarVT:$sx), DataVT:$vy, | |
| (MaskVT true_mask), | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"rvl") | |
| ScalarVT:$sx, $vy, $avl)>; | |
| // Masked, broadcast. | |
| def : Pat<(OpNode | |
| (any_broadcast ScalarVT:$sx), DataVT:$vy, | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"rvml") | |
| ScalarVT:$sx, $vy, $mask, $avl)>; | |
| } | |
| multiclass Binary_vr<SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru, broadcast. | |
| def : Pat<(vvp_select | |
| (OpNode | |
| DataVT:$vx, | |
| (any_broadcast ScalarVT:$sy), | |
| (MaskVT srcvalue), | |
| (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$pivot), | |
| (!cast<Instruction>(OpBaseName#"vrml_v") | |
| $vx, | |
| ScalarVT:$sy, | |
| $mask, | |
| $pivot, | |
| $vfalse)>; | |
| // Unmasked, broadcast. | |
| def : Pat<(OpNode | |
| DataVT:$vx, (any_broadcast ScalarVT:$sy), | |
| (MaskVT true_mask), | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vrl") | |
| $vx, ScalarVT:$sy, $avl)>; | |
| // Masked, broadcast. | |
| def : Pat<(OpNode | |
| DataVT:$vx, (any_broadcast ScalarVT:$sy), | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vrml") | |
| $vx, ScalarVT:$sy, $mask, $avl)>; | |
| } | |
| multiclass Binary_vv<SDPatternOperator OpNode, | |
| ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru, broadcast. | |
| def : Pat<(vvp_select | |
| (OpNode | |
| DataVT:$vx, | |
| DataVT:$vy, | |
| (MaskVT srcvalue), | |
| (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$pivot), | |
| (!cast<Instruction>(OpBaseName#"vvml_v") | |
| $vx, | |
| $vy, | |
| $mask, | |
| $pivot, | |
| $vfalse)>; | |
| // Masked with select. | |
| // TODO | |
| // Unmasked. | |
| def : Pat<(OpNode | |
| DataVT:$vx, DataVT:$vy, | |
| (MaskVT true_mask), | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvl") | |
| $vx, $vy, $avl)>; | |
| // Masked. | |
| def : Pat<(OpNode | |
| DataVT:$vx, DataVT:$vy, | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvml") | |
| $vx, $vy, $mask, $avl)>; | |
| } | |
| multiclass Binary_rv_vv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, | |
| string OpBaseName> { | |
| defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>; | |
| } | |
| multiclass Binary_vr_vv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, | |
| string OpBaseName> { | |
| defm : Binary_vr<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| defm : Binary_vv<OpNode, DataVT, MaskVT, OpBaseName>; | |
| } | |
| multiclass Binary_rv_vr_vv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, ValueType MaskVT, | |
| string OpBaseName> { | |
| defm : Binary_rv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| defm : Binary_vr_vv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| } | |
| // Expand both 64bit and 32 bit variant (256 elements) | |
| multiclass Binary_rv_vv_ShortLong< | |
| SDPatternOperator OpNode, | |
| ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, | |
| ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { | |
| defm : Binary_rv_vv<OpNode, | |
| LongScalarVT, LongDataVT, v256i1, | |
| LongOpBaseName>; | |
| defm : Binary_rv_vv<OpNode, | |
| ShortScalarVT, ShortDataVT, v256i1, | |
| ShortOpBaseName>; | |
| } | |
| multiclass Binary_vr_vv_ShortLong< | |
| SDPatternOperator OpNode, | |
| ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, | |
| ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { | |
| defm : Binary_vr_vv<OpNode, | |
| LongScalarVT, LongDataVT, v256i1, | |
| LongOpBaseName>; | |
| defm : Binary_vr_vv<OpNode, | |
| ShortScalarVT, ShortDataVT, v256i1, | |
| ShortOpBaseName>; | |
| } | |
| multiclass Binary_rv_vr_vv_ShortLong< | |
| SDPatternOperator OpNode, | |
| ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, | |
| ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { | |
| defm : Binary_rv_vr_vv<OpNode, | |
| LongScalarVT, LongDataVT, v256i1, | |
| LongOpBaseName>; | |
| defm : Binary_rv_vr_vv<OpNode, | |
| ShortScalarVT, ShortDataVT, v256i1, | |
| ShortOpBaseName>; | |
| } | |
| defm : Binary_rv_vv_ShortLong<c_vvp_add, | |
| i64, v256i64, "VADDSL", | |
| i32, v256i32, "VADDSWSX">; | |
| defm : Binary_rv_vv_ShortLong<vvp_sub, | |
| i64, v256i64, "VSUBSL", | |
| i32, v256i32, "VSUBSWSX">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_mul, | |
| i64, v256i64, "VMULSL", | |
| i32, v256i32, "VMULSWSX">; | |
| defm : Binary_rv_vr_vv_ShortLong<vvp_sdiv, | |
| i64, v256i64, "VDIVSL", | |
| i32, v256i32, "VDIVSWSX">; | |
| defm : Binary_rv_vr_vv_ShortLong<vvp_udiv, | |
| i64, v256i64, "VDIVUL", | |
| i32, v256i32, "VDIVUW">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_and, | |
| i64, v256i64, "VAND", | |
| i32, v256i32, "PVANDLO">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_or, | |
| i64, v256i64, "VOR", | |
| i32, v256i32, "PVORLO">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_xor, | |
| i64, v256i64, "VXOR", | |
| i32, v256i32, "PVXORLO">; | |
| defm : Binary_vr_vv_ShortLong<vvp_shl, | |
| i64, v256i64, "VSLL", | |
| i32, v256i32, "PVSLLLO">; | |
| defm : Binary_vr_vv_ShortLong<vvp_sra, | |
| i64, v256i64, "VSRAL", | |
| i32, v256i32, "PVSRALO">; | |
| defm : Binary_vr_vv_ShortLong<vvp_srl, | |
| i64, v256i64, "VSRL", | |
| i32, v256i32, "PVSRLLO">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_fadd, | |
| f64, v256f64, "VFADDD", | |
| f32, v256f32, "PVFADDUP">; | |
| defm : Binary_rv_vv_ShortLong<c_vvp_fmul, | |
| f64, v256f64, "VFMULD", | |
| f32, v256f32, "PVFMULUP">; | |
| defm : Binary_rv_vv_ShortLong<vvp_fsub, | |
| f64, v256f64, "VFSUBD", | |
| f32, v256f32, "PVFSUBUP">; | |
| defm : Binary_rv_vr_vv_ShortLong<vvp_fdiv, | |
| f64, v256f64, "VFDIVD", | |
| f32, v256f32, "VFDIVS">; | |
| defm : Binary_rv_vv<c_vvp_and, | |
| i64, v512i32, v512i1, "PVAND">; | |
| defm : Binary_rv_vv<c_vvp_or, | |
| i64, v512i32, v512i1, "PVOR">; | |
| defm : Binary_rv_vv<c_vvp_xor, | |
| i64, v512i32, v512i1, "PVXOR">; | |
| defm : Binary_rv_vv<c_vvp_add, | |
| i64, v512i32, v512i1, "PVADDU">; | |
| defm : Binary_rv_vv<vvp_sub, | |
| i64, v512i32, v512i1, "PVSUBU">; | |
| defm : Binary_vr_vv<vvp_srl, | |
| i64, v512i32, v512i1, "PVSRL">; | |
| defm : Binary_vr_vv<vvp_sra, | |
| i64, v512i32, v512i1, "PVSRA">; | |
| defm : Binary_vr_vv<vvp_shl, | |
| i64, v512i32, v512i1, "PVSLL">; | |
| defm : Binary_rv_vv<c_vvp_fadd, | |
| i64, v512f32, v512i1, "PVFADD">; | |
| defm : Binary_rv_vv<c_vvp_fmul, | |
| i64, v512f32, v512i1, "PVFMUL">; | |
| defm : Binary_rv_vv<vvp_fsub, | |
| i64, v512f32, v512i1, "PVFSUB">; | |
| multiclass Ternary_vvv< | |
| SDPatternOperator OpNode, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru. | |
| def : Pat<(vvp_select | |
| (OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz, | |
| (MaskVT srcvalue), (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvvml_v") | |
| $vx, $vy, $vz, $mask, $avl, $vfalse)>; | |
| // Unmasked. | |
| def : Pat<(OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz, | |
| (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvvl") | |
| $vx, $vy, $vz, $avl)>; | |
| // Masked. | |
| def : Pat<(OpNode DataVT:$vx, DataVT:$vy, DataVT:$vz, | |
| MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvvml") | |
| $vx, $vy, $vz, $mask, $avl)>; | |
| } | |
| multiclass Ternary_rvv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru, broadcast first. | |
| def : Pat<(vvp_select | |
| (OpNode | |
| (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz, | |
| (MaskVT srcvalue), (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"rvvml_v") | |
| $sx, $vy, $vz, $mask, $avl, $vfalse)>; | |
| // Unmasked, broadcast first. | |
| def : Pat<(OpNode | |
| (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz, | |
| (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"rvvl") | |
| $sx, $vy, $vz, $avl)>; | |
| // Masked, broadcast first. | |
| def : Pat<(OpNode | |
| (any_broadcast ScalarVT:$sx), DataVT:$vy, DataVT:$vz, | |
| MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"rvvml") | |
| $sx, $vy, $vz, $mask, $avl)>; | |
| } | |
| multiclass Ternary_vrv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| // Masked with passthru, broadcast second. | |
| def : Pat<(vvp_select | |
| (OpNode | |
| DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz, | |
| (MaskVT srcvalue), (i32 srcvalue)), | |
| DataVT:$vfalse, | |
| MaskVT:$mask, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vrvml_v") | |
| $vx, $sy, $vz, | |
| $mask, $avl, $vfalse)>; | |
| // Unmasked, broadcast second. | |
| def : Pat<(OpNode | |
| DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz, | |
| (MaskVT true_mask), i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vrvl") | |
| $vx, $sy, $vz, $avl)>; | |
| // Masked, broadcast second. | |
| def : Pat<(OpNode | |
| DataVT:$vx, (any_broadcast ScalarVT:$sy), DataVT:$vz, | |
| MaskVT:$mask, i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vrvml") | |
| $vx, $sy, $vz, $mask, $avl)>; | |
| } | |
| multiclass Ternary_rvv_vrv_vvv< | |
| SDPatternOperator OpNode, | |
| ValueType ScalarVT, ValueType DataVT, | |
| ValueType MaskVT, string OpBaseName> { | |
| defm : Ternary_rvv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| defm : Ternary_vrv<OpNode, ScalarVT, DataVT, MaskVT, OpBaseName>; | |
| defm : Ternary_vvv<OpNode, DataVT, MaskVT, OpBaseName>; | |
| } | |
| // Expand both 64bit and 32 bit variant (256 elements) | |
| multiclass Ternary_ShortLong< | |
| SDPatternOperator OpNode, | |
| ValueType LongScalarVT, ValueType LongDataVT, string LongOpBaseName, | |
| ValueType ShortScalarVT, ValueType ShortDataVT, string ShortOpBaseName> { | |
| defm : Ternary_rvv_vrv_vvv<OpNode, LongScalarVT, LongDataVT, | |
| v256i1, LongOpBaseName>; | |
| defm : Ternary_rvv_vrv_vvv<OpNode, ShortScalarVT, ShortDataVT, | |
| v256i1, ShortOpBaseName>; | |
| } | |
| defm : Ternary_ShortLong<c_vvp_ffma, | |
| f64, v256f64, "VFMADD", f32, v256f32, "VFMADS">; | |
| defm : Ternary_rvv_vrv_vvv<c_vvp_ffma, | |
| i64, v512f32, v512i1, "PVFMAD">; | |
| multiclass Merge_mvv< | |
| SDPatternOperator OpNode, | |
| ValueType DataVT, ValueType MaskVT, | |
| string OpBaseName> { | |
| // Masked. | |
| def : Pat<(OpNode | |
| DataVT:$vtrue, DataVT:$vfalse, | |
| MaskVT:$vm, | |
| i32:$avl), | |
| (!cast<Instruction>(OpBaseName#"vvml_v") | |
| $vfalse, $vtrue, $vm, $avl, $vfalse)>; | |
| } | |
| multiclass Merge_mvv_ShortLong< | |
| SDPatternOperator OpNode, | |
| ValueType LongDataVT, ValueType ShortDataVT, | |
| string OpBaseName> { | |
| defm : Merge_mvv<OpNode, | |
| LongDataVT, v256i1, | |
| OpBaseName>; | |
| defm : Merge_mvv<OpNode, | |
| ShortDataVT, v256i1, | |
| OpBaseName>; | |
| } | |
| defm : Merge_mvv_ShortLong<vvp_select, | |
| v256f64, | |
| v256f32, "VMRG">; | |
| defm : Merge_mvv_ShortLong<vvp_select, | |
| v256i64, | |
| v256i32, "VMRG">; | |
| multiclass Set_CC<ValueType DataVT, string FmkBaseName, string CmpBaseName, SDPatternOperator CCMatcher, SDNodeXForm CCConv> { | |
| // Unmasked. | |
| def : Pat<(v256i1 (vvp_setcc | |
| DataVT:$LHS, DataVT:$RHS, CCMatcher:$cond, (v256i1 true_mask), i32:$vl)), | |
| (!cast<Instruction>(FmkBaseName#"vl") | |
| (CCConv $cond), | |
| (!cast<Instruction>(CmpBaseName#"vvl") | |
| $LHS, $RHS, $vl), | |
| $vl)>; | |
| // Masked. | |
| def : Pat<(v256i1 (vvp_setcc | |
| DataVT:$LHS, DataVT:$RHS, CCMatcher:$cond, v256i1:$vm, i32:$vl)), | |
| (!cast<Instruction>(FmkBaseName#"vml") | |
| (CCConv $cond), | |
| (!cast<Instruction>(CmpBaseName#"vvl") | |
| $LHS, $RHS, $vl), | |
| $vm, $vl)>; | |
| } | |
| defm : Set_CC<v256i64,"VFMKL","VCMPUL",CCUIOp,icond2cc>; | |
| defm : Set_CC<v256i64,"VFMKL","VCMPSL",CCSIOp,icond2cc>; | |
| defm : Set_CC<v256f64,"VFMKL","VFCMPD",cond,fcond2cc>; | |
| defm : Set_CC<v256i32,"VFMKW","VCMPUW",CCUIOp,icond2cc>; | |
| defm : Set_CC<v256i32,"VFMKW","VCMPSWZX",CCSIOp,icond2cc>; | |
| defm : Set_CC<v256f32,"VFMKS","VFCMPS",cond,fcond2cc>; | |
| multiclass Reduce_GenericInt<ValueType VectorVT, | |
| RegisterClass ResRC, ValueType ResVT, | |
| string VVPRedOp, string RedInstName> { | |
| // Unmasked. | |
| def : Pat <(ResVT (!cast<SDPatternOperator>("vvp_reduce_"#VVPRedOp) | |
| VectorVT:$vx, (v256i1 true_mask), i32:$vl)), | |
| (COPY_TO_REGCLASS | |
| (!cast<Instruction>("LVSvi") | |
| (!cast<Instruction>(RedInstName#"vl") $vx, $vl), 0), | |
| ResRC)>; | |
| // Masked. | |
| def : Pat <(ResVT (!cast<SDPatternOperator>("vvp_reduce_"#VVPRedOp) | |
| VectorVT:$vx, v256i1:$vm, i32:$vl)), | |
| (COPY_TO_REGCLASS | |
| (!cast<Instruction>("LVSvi") | |
| (!cast<Instruction>(RedInstName#"vml") $vx, $vm, $vl), 0), | |
| ResRC)>; | |
| } | |
| multiclass IntReduce_ShortLong<ValueType VectorVT, | |
| RegisterClass ResRC, ValueType ResVT, | |
| string SumSuffix, string MinMaxSuffix> { | |
| defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "or", "VROR">; | |
| defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "and", "VRAND">; | |
| defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "xor", "VRXOR">; | |
| defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "add", "VSUM"#SumSuffix>; | |
| defm: Reduce_GenericInt<VectorVT, ResRC, ResVT, "smax", "VRMAX"#MinMaxSuffix>; | |
| } | |
| defm: IntReduce_ShortLong<v256i64, I64, i64, "L","SLFST">; | |
| defm: IntReduce_ShortLong<v256i32, I32, i32, "WSX","SWFSTSX">; |