diff --git a/src/cmd/compile/internal/arm64/simdssa_sve.go b/src/cmd/compile/internal/arm64/simdssa_sve.go index 6061c9c3e6ac41..5f184b09c94965 100644 --- a/src/cmd/compile/internal/arm64/simdssa_sve.go +++ b/src/cmd/compile/internal/arm64/simdssa_sve.go @@ -47,6 +47,50 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { case ssaop.OpARM64ZCMPGTS: p = simdZ2kk(s, v, arm64.ARNG_S) + case ssaop.OpARM64ZADDMergingB, + ssaop.OpARM64ZSQADDMergingB, + ssaop.OpARM64ZUQADDMergingB: + p = simdZ2kvPred(s, v, arm64.ARNG_B) + + case ssaop.OpARM64ZFADDMergingD, + ssaop.OpARM64ZADDMergingD, + ssaop.OpARM64ZSQADDMergingD, + ssaop.OpARM64ZUQADDMergingD: + p = simdZ2kvPred(s, v, arm64.ARNG_D) + + case ssaop.OpARM64ZADDMergingH, + ssaop.OpARM64ZSQADDMergingH, + ssaop.OpARM64ZUQADDMergingH: + p = simdZ2kvPred(s, v, arm64.ARNG_H) + + case ssaop.OpARM64ZFADDMergingS, + ssaop.OpARM64ZADDMergingS, + ssaop.OpARM64ZSQADDMergingS, + ssaop.OpARM64ZUQADDMergingS: + p = simdZ2kvPred(s, v, arm64.ARNG_S) + + case ssaop.OpARM64ZADDMergingPrefixedB, + ssaop.OpARM64ZSQADDMergingPrefixedB, + ssaop.OpARM64ZUQADDMergingPrefixedB: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_B) + + case ssaop.OpARM64ZFADDMergingPrefixedD, + ssaop.OpARM64ZADDMergingPrefixedD, + ssaop.OpARM64ZSQADDMergingPrefixedD, + ssaop.OpARM64ZUQADDMergingPrefixedD: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_D) + + case ssaop.OpARM64ZADDMergingPrefixedH, + ssaop.OpARM64ZSQADDMergingPrefixedH, + ssaop.OpARM64ZUQADDMergingPrefixedH: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_H) + + case ssaop.OpARM64ZFADDMergingPrefixedS, + ssaop.OpARM64ZADDMergingPrefixedS, + ssaop.OpARM64ZSQADDMergingPrefixedS, + ssaop.OpARM64ZUQADDMergingPrefixedS: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_S) + default: // Unknown reg shape return false diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go index 16fd3c4683ac47..70fc8f60d7d3f2 100644 --- a/src/cmd/compile/internal/arm64/ssa.go +++ b/src/cmd/compile/internal/arm64/ssa.go @@ -829,6 +829,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p.From.Offset = v.AuxInt p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() + case ssaop.OpARM64ZSELB: + simdZ2kv(s, v, arm64.ARNG_B) + case ssaop.OpARM64ZSELH: + simdZ2kv(s, v, arm64.ARNG_H) + case ssaop.OpARM64ZSELS: + simdZ2kv(s, v, arm64.ARNG_S) + case ssaop.OpARM64ZSELD: + simdZ2kv(s, v, arm64.ARNG_D) case ssaop.OpARM64PWHILELTB: simdPWHILELT(s, v, arm64.ARNG_B) case ssaop.OpARM64PWHILELTH: @@ -2122,6 +2130,100 @@ func simdZ21(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { return p } +// simdZ2kv emits an SVE instruction that takes a predicate as a plain data +// operand rather than a governing predicate, e.g. ZSEL Z1.B, Z0.B, P0, Z2.B. +// Unlike a predicated instruction it is constructive: the destination is +// independent of the sources, so it needs no MOVPRFX. SSA provides arg0=x (kept +// where the predicate is true), arg1=y (kept where false) and arg2=the predicate. +func simdZ2kv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(v.Args[1].Reg(), arng) // Zm + p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn + p.AddRestSourceReg(v.Args[2].Reg()) // Pv, unqualified + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(v.Reg(), arng) // Zd + return p +} + +// simdZ2kvPred emits an SVE predicated binary operation, e.g. ZADD Z1.B, Z0.B, P0.M, +// Z0.B. These instructions are destructive: the destination is also the first +// source. How the destination is put in place depends on what the register +// allocator chose: +// +// - dst == arg0: already destructive-ready, emit the instruction alone. +// - dst == arg1: only reachable for a commutative operation (a +// non-commutative one is marked resultInArg0, which pins dst to arg0), so +// swap the sources and it becomes the case above. +// - anything else: prefix MOVPRFX, which hints the hardware to fuse the pair +// into one constructive operation and leaves both sources intact. +func simdZ2kvPred(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + x, y := v.Args[0].Reg(), v.Args[1].Reg() + d := v.Reg() + switch d { + case x: + case y: + x, y = y, x + default: + mp := s.Prog(arm64.AZMOVPRFX) + mp.From.Type = obj.TYPE_REG + mp.From.Reg = pzreg(x) + mp.To.Type = obj.TYPE_REG + mp.To.Reg = pzreg(d) + } + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(y, arng) // Zm + p.AddRestSourceReg(zregArng(d, arng)) // Zdn + p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M)) // Pg/M + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(d, arng) // Zdn + return p +} + +// simdZ3kvPredResultInArg0 emits an SVE merging-predicated binary operation +// whose inactive lanes come from a value that is neither of its sources, e.g. +// x.Add(y).IfElse(mask, z). SSA provides arg0=z, arg1=x, arg2=y, arg3=mask, and +// resultInArg0 puts z in the destination register. +// +// ZMOVPRFX Zx, Pg/M, Zd // Zd = x on the active lanes, z on the rest +// ZADD Zy, Zd, Pg/M, Zd // Zd = x+y on the active lanes, z on the rest +// +// The prefix must be the predicated MOVPRFX, not the unpredicated one: the +// whole-register form would leave x rather than z in the inactive lanes. +// +// A prefixed instruction may not name its destination in any operand position +// other than the destructive one, so Zd must differ from the operation's Zm. +// Only a commutative operation gets this lowering (see sveMergingPrefixedOp in +// simdgen), so when the register allocator puts the destination on one source — +// which it can only do when z is that source — the other one becomes Zm and the +// prefix is unnecessary. +func simdZ3kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + d := v.Reg() + x, y := v.Args[1].Reg(), v.Args[2].Reg() + pg := v.Args[3].Reg() + zn, zm := x, y + if d == zm { + zn, zm = zm, zn + } + if d != zn { + mp := s.Prog(arm64.AZMOVPRFX) + mp.From.Type = obj.TYPE_REG + mp.From.Reg = zregArng(zn, arng) + mp.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) + mp.To.Type = obj.TYPE_REG + mp.To.Reg = zregArng(d, arng) + } + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(zm, arng) // Zm + p.AddRestSourceReg(zregArng(d, arng)) // Zdn + p.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) // Pg/M + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(d, arng) // Zdn + return p +} + // simdPWHILELT emits a PWHILELT that fills a predicate with lanes [lo,hi) set for // the given element arrangement, e.g. PWHILELT R0, R1, P0.B. SSA provides // arg0=lo, arg1=hi. diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64.rules b/src/cmd/compile/internal/ssa/_gen/ARM64.rules index 6bde3d674a439f..be80773590db56 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/_gen/ARM64.rules @@ -1905,5 +1905,18 @@ // These are written by hand until the predicated ops are generated by simdgen // in the mask CL; at that point delete them and generate instead. (Count8s r) => (Select0 (PWHILELTB (MOVDconst [0]) r)) + +// SVE per-element select, backing IfElse and Masked. +(IfElseInt8s x mask y) => (ZSELB x y mask) +(IfElseUint8s x mask y) => (ZSELB x y mask) +(IfElseInt16s x mask y) => (ZSELH x y mask) +(IfElseUint16s x mask y) => (ZSELH x y mask) +(IfElseInt32s x mask y) => (ZSELS x y mask) +(IfElseUint32s x mask y) => (ZSELS x y mask) +(IfElseFloat32s x mask y) => (ZSELS x y mask) +(IfElseInt64s x mask y) => (ZSELD x y mask) +(IfElseUint64s x mask y) => (ZSELD x y mask) +(IfElseFloat64s x mask y) => (ZSELD x y mask) + (LoadMasked8 ptr mask mem) && t.Size() == 32 => (ZLD1BPredload ptr mask mem) (StoreMasked8 {t} ptr mask val mem) && t.Size() == 32 => (ZST1BPredstore ptr val mask mem) diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go index 17c4e1c41b2b0e..e0033283c13b45 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go @@ -204,6 +204,8 @@ func init() { fp2flags = regInfo{inputs: []regMask{fp, fp}} fp1flags = regInfo{inputs: []regMask{fp}} fp2predpred = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{pred}} + fp2predfp = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{fp}} + fp3predfp = regInfo{inputs: []regMask{fp, fp, fp, pred}, outputs: []regMask{fp}} predload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{pred}} predstore = regInfo{inputs: []regMask{gpspsbg, pred}} fpload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{fp}} @@ -839,6 +841,10 @@ func init() { // scalable Z bank reuses the fp register masks. {name: "ZLDRload", argLength: 2, reg: fpload, aux: "SymOff", asm: "ZLDR", typ: "Vec256", faultOnNilArg0: true, symEffect: "Read"}, // load from arg0 + auxInt + aux. arg1=mem. {name: "ZSTRstore", argLength: 3, reg: fpstore, aux: "SymOff", asm: "ZSTR", faultOnNilArg0: true, symEffect: "Write"}, // store arg1 to arg0 + auxInt + aux. arg2=mem. + {name: "ZSELB", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELH", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELS", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELD", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. {name: "PLDRload", argLength: 2, reg: predload, aux: "SymOff", asm: "PLDR", typ: "Mask", faultOnNilArg0: true, symEffect: "Read"}, // load a predicate from arg0 + auxInt + aux. arg1=mem. {name: "PSTRstore", argLength: 3, reg: predstore, aux: "SymOff", asm: "PSTR", faultOnNilArg0: true, symEffect: "Write"}, // store predicate arg1 to arg0 + auxInt + aux. arg2=mem. // PPFALSEB sets every bit of a predicate false, it's the zero value of a predicate. @@ -896,7 +902,7 @@ func init() { pkg: "cmd/internal/obj/arm64", genfile: "../../arm64/ssa.go", genSIMDfile: "../../arm64/simdssa.go ../../arm64/simdssa_sve.go", - ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred)...), + ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred, fp2predfp, fp2predfp, fp3predfp)...), blocks: blocks, regnames: regNamesARM64, ParamIntRegNames: "R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15", diff --git a/src/cmd/compile/internal/ssa/_gen/genericOps.go b/src/cmd/compile/internal/ssa/_gen/genericOps.go index a11a686fd36481..860187b9ad2a31 100644 --- a/src/cmd/compile/internal/ssa/_gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/genericOps.go @@ -746,6 +746,22 @@ var genericOps = []opData{ {name: "ScalableVectorLen", argLength: 0}, // SVE runtime vector length in bytes {name: "Count8s", argLength: 1}, // arg0 = active byte count; builds an SVE predicate over that many byte lanes + + // IfElse selects per element between two scalable vectors under a predicate. + // It backs both the IfElse and (against a zero vector) the Masked method, and + // is written by hand rather than derived from the ISA because SEL is + // bit-pattern-agnostic: there is no float-lane encoding of it to unify with. + // arg0 = x, arg1 = predicate, arg2 = y (taken where the predicate is false). + {name: "IfElseInt8s", argLength: 3}, + {name: "IfElseUint8s", argLength: 3}, + {name: "IfElseInt16s", argLength: 3}, + {name: "IfElseUint16s", argLength: 3}, + {name: "IfElseInt32s", argLength: 3}, + {name: "IfElseUint32s", argLength: 3}, + {name: "IfElseFloat32s", argLength: 3}, + {name: "IfElseInt64s", argLength: 3}, + {name: "IfElseUint64s", argLength: 3}, + {name: "IfElseFloat64s", argLength: 3}, } // kind controls successors implicit exit diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules index dc7905a5eeb866..d38949935e387a 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules @@ -22,3 +22,45 @@ (GreaterInt32s x y) => (ZCMPGTS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) (GreaterInt64s x y) => (ZCMPGTD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) (GreaterInt8s x y) => (ZCMPGTB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(ZSELB (ZADDB x y) x mask) => (ZADDMergingB x y mask) +(ZSELB (ZADDB x y) y mask) => (ZADDMergingB y x mask) +(ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask) +(ZSELB (ZSQADDB x y) x mask) => (ZSQADDMergingB x y mask) +(ZSELB (ZSQADDB x y) y mask) => (ZSQADDMergingB y x mask) +(ZSELB (ZSQADDB x y) z mask) => (ZSQADDMergingPrefixedB z x y mask) +(ZSELB (ZUQADDB x y) x mask) => (ZUQADDMergingB x y mask) +(ZSELB (ZUQADDB x y) y mask) => (ZUQADDMergingB y x mask) +(ZSELB (ZUQADDB x y) z mask) => (ZUQADDMergingPrefixedB z x y mask) +(ZSELD (ZADDD x y) x mask) => (ZADDMergingD x y mask) +(ZSELD (ZADDD x y) y mask) => (ZADDMergingD y x mask) +(ZSELD (ZADDD x y) z mask) => (ZADDMergingPrefixedD z x y mask) +(ZSELD (ZFADDD x y) x mask) => (ZFADDMergingD x y mask) +(ZSELD (ZFADDD x y) y mask) => (ZFADDMergingD y x mask) +(ZSELD (ZFADDD x y) z mask) => (ZFADDMergingPrefixedD z x y mask) +(ZSELD (ZSQADDD x y) x mask) => (ZSQADDMergingD x y mask) +(ZSELD (ZSQADDD x y) y mask) => (ZSQADDMergingD y x mask) +(ZSELD (ZSQADDD x y) z mask) => (ZSQADDMergingPrefixedD z x y mask) +(ZSELD (ZUQADDD x y) x mask) => (ZUQADDMergingD x y mask) +(ZSELD (ZUQADDD x y) y mask) => (ZUQADDMergingD y x mask) +(ZSELD (ZUQADDD x y) z mask) => (ZUQADDMergingPrefixedD z x y mask) +(ZSELH (ZADDH x y) x mask) => (ZADDMergingH x y mask) +(ZSELH (ZADDH x y) y mask) => (ZADDMergingH y x mask) +(ZSELH (ZADDH x y) z mask) => (ZADDMergingPrefixedH z x y mask) +(ZSELH (ZSQADDH x y) x mask) => (ZSQADDMergingH x y mask) +(ZSELH (ZSQADDH x y) y mask) => (ZSQADDMergingH y x mask) +(ZSELH (ZSQADDH x y) z mask) => (ZSQADDMergingPrefixedH z x y mask) +(ZSELH (ZUQADDH x y) x mask) => (ZUQADDMergingH x y mask) +(ZSELH (ZUQADDH x y) y mask) => (ZUQADDMergingH y x mask) +(ZSELH (ZUQADDH x y) z mask) => (ZUQADDMergingPrefixedH z x y mask) +(ZSELS (ZADDS x y) x mask) => (ZADDMergingS x y mask) +(ZSELS (ZADDS x y) y mask) => (ZADDMergingS y x mask) +(ZSELS (ZADDS x y) z mask) => (ZADDMergingPrefixedS z x y mask) +(ZSELS (ZFADDS x y) x mask) => (ZFADDMergingS x y mask) +(ZSELS (ZFADDS x y) y mask) => (ZFADDMergingS y x mask) +(ZSELS (ZFADDS x y) z mask) => (ZFADDMergingPrefixedS z x y mask) +(ZSELS (ZSQADDS x y) x mask) => (ZSQADDMergingS x y mask) +(ZSELS (ZSQADDS x y) y mask) => (ZSQADDMergingS y x mask) +(ZSELS (ZSQADDS x y) z mask) => (ZSQADDMergingPrefixedS z x y mask) +(ZSELS (ZUQADDS x y) x mask) => (ZUQADDMergingS x y mask) +(ZSELS (ZUQADDS x y) y mask) => (ZUQADDMergingS y x mask) +(ZSELS (ZUQADDS x y) z mask) => (ZUQADDMergingPrefixedS z x y mask) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go index 98a2164e1fb8d4..fd3453977bda51 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go @@ -2,25 +2,53 @@ package main -func simdARM64SVEOps(z11, z21, z2kk regInfo) []opData { +func simdARM64SVEOps(z11, z21, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []opData { return []opData{ {name: "ZADDB", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDD", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDH", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDS", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZCMPGTB", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTD", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTH", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTS", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZFADDD", argLength: 2, reg: z21, asm: "ZFADD", commutative: true, typ: "Vec256"}, + {name: "ZFADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZFADD", commutative: true, typ: "Vec256"}, + {name: "ZFADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZFADD", typ: "Vec256", resultInArg0: true}, + {name: "ZFADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZFADD", typ: "Vec256", resultInArg0: true}, + {name: "ZFADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZFADD", commutative: true, typ: "Vec256"}, {name: "ZFADDS", argLength: 2, reg: z21, asm: "ZFADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDB", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDD", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDH", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDS", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDB", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDD", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDH", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDS", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, } } diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index debf9d284726f2..a1b687e63b8487 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -4715,6 +4715,10 @@ const ( OpARM64VMOVI16B OpARM64ZLDRload OpARM64ZSTRstore + OpARM64ZSELB + OpARM64ZSELH + OpARM64ZSELS + OpARM64ZSELD OpARM64PLDRload OpARM64PSTRstore OpARM64PPFALSEB @@ -5039,20 +5043,48 @@ const ( OpARM64ZADDB OpARM64ZADDD OpARM64ZADDH + OpARM64ZADDMergingB + OpARM64ZADDMergingD + OpARM64ZADDMergingH + OpARM64ZADDMergingPrefixedB + OpARM64ZADDMergingPrefixedD + OpARM64ZADDMergingPrefixedH + OpARM64ZADDMergingPrefixedS + OpARM64ZADDMergingS OpARM64ZADDS OpARM64ZCMPGTB OpARM64ZCMPGTD OpARM64ZCMPGTH OpARM64ZCMPGTS OpARM64ZFADDD + OpARM64ZFADDMergingD + OpARM64ZFADDMergingPrefixedD + OpARM64ZFADDMergingPrefixedS + OpARM64ZFADDMergingS OpARM64ZFADDS OpARM64ZSQADDB OpARM64ZSQADDD OpARM64ZSQADDH + OpARM64ZSQADDMergingB + OpARM64ZSQADDMergingD + OpARM64ZSQADDMergingH + OpARM64ZSQADDMergingPrefixedB + OpARM64ZSQADDMergingPrefixedD + OpARM64ZSQADDMergingPrefixedH + OpARM64ZSQADDMergingPrefixedS + OpARM64ZSQADDMergingS OpARM64ZSQADDS OpARM64ZUQADDB OpARM64ZUQADDD OpARM64ZUQADDH + OpARM64ZUQADDMergingB + OpARM64ZUQADDMergingD + OpARM64ZUQADDMergingH + OpARM64ZUQADDMergingPrefixedB + OpARM64ZUQADDMergingPrefixedD + OpARM64ZUQADDMergingPrefixedH + OpARM64ZUQADDMergingPrefixedS + OpARM64ZUQADDMergingS OpARM64ZUQADDS OpLOONG64NEGV @@ -6962,6 +6994,16 @@ const ( OpIsNaNFloat64x8 OpScalableVectorLen OpCount8s + OpIfElseInt8s + OpIfElseUint8s + OpIfElseInt16s + OpIfElseUint16s + OpIfElseInt32s + OpIfElseUint32s + OpIfElseFloat32s + OpIfElseInt64s + OpIfElseUint64s + OpIfElseFloat64s OpAESDecryptLastRoundUint8x16 OpAESDecryptLastRoundUint8x32 OpAESDecryptLastRoundUint8x64 @@ -81052,6 +81094,66 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZSELB", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELH", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELS", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELD", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "PLDRload", AuxType: AuxTypeSymOff, @@ -85621,6 +85723,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZADDS", ArgLen: 2, @@ -85711,6 +85945,72 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZFADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZFADDS", ArgLen: 2, @@ -85771,6 +86071,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZSQADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZSQADDS", ArgLen: 2, @@ -85831,6 +86263,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZUQADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZUQADDS", ArgLen: 2, @@ -109275,6 +109839,56 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 1, Generic: true, }, + { + Name: "IfElseInt8s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint8s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt16s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint16s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseFloat32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt64s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint64s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseFloat64s", + ArgLen: 3, + Generic: true, + }, { Name: "AESDecryptLastRoundUint8x16", ArgLen: 2, diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go index 0d93a9ee5c63bf..6a0c4e41a32368 100644 --- a/src/cmd/compile/internal/ssagen/intrinsics.go +++ b/src/cmd/compile/internal/ssagen/intrinsics.go @@ -1700,6 +1700,24 @@ func initIntrinsics(cfg *intrinsicBuildConfig) { addF(simdPackage, "load"+t.name+"Part", sveLoadPart(t.bytes), sys.ARM64) addF(simdPackage, t.name+".storePart", sveStorePart(t.bytes), sys.ARM64) } + // IfElse backs both the IfElse and Masked methods on every scalable vector. + for _, t := range []struct { + name string + op ssaop.Op + }{ + {"Int8s", ssaop.OpIfElseInt8s}, + {"Uint8s", ssaop.OpIfElseUint8s}, + {"Int16s", ssaop.OpIfElseInt16s}, + {"Uint16s", ssaop.OpIfElseUint16s}, + {"Int32s", ssaop.OpIfElseInt32s}, + {"Uint32s", ssaop.OpIfElseUint32s}, + {"Float32s", ssaop.OpIfElseFloat32s}, + {"Int64s", ssaop.OpIfElseInt64s}, + {"Uint64s", ssaop.OpIfElseUint64s}, + {"Float64s", ssaop.OpIfElseFloat64s}, + } { + addF(simdPackage, t.name+".IfElse", opLen3(t.op, types.TypeVec256), sys.ARM64) + } addF(simdPackage, "ClearAVXUpperBits", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { diff --git a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go index be73f1e02f3039..b9f150810ce6d7 100644 --- a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go +++ b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go @@ -519,6 +519,14 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpARM64XORshiftRL(v) case ssaop.OpARM64XORshiftRO: return rewriteValue_OpARM64XORshiftRO(v) + case ssaop.OpARM64ZSELB: + return rewriteValue_OpARM64ZSELB(v) + case ssaop.OpARM64ZSELD: + return rewriteValue_OpARM64ZSELD(v) + case ssaop.OpARM64ZSELH: + return rewriteValue_OpARM64ZSELH(v) + case ssaop.OpARM64ZSELS: + return rewriteValue_OpARM64ZSELS(v) case ssaop.OpAbs: v.Op = ssaop.OpARM64FABSD return true @@ -1331,6 +1339,26 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpHmul64u: v.Op = ssaop.OpARM64UMULH return true + case ssaop.OpIfElseFloat32s: + return rewriteValue_OpIfElseFloat32s(v) + case ssaop.OpIfElseFloat64s: + return rewriteValue_OpIfElseFloat64s(v) + case ssaop.OpIfElseInt16s: + return rewriteValue_OpIfElseInt16s(v) + case ssaop.OpIfElseInt32s: + return rewriteValue_OpIfElseInt32s(v) + case ssaop.OpIfElseInt64s: + return rewriteValue_OpIfElseInt64s(v) + case ssaop.OpIfElseInt8s: + return rewriteValue_OpIfElseInt8s(v) + case ssaop.OpIfElseUint16s: + return rewriteValue_OpIfElseUint16s(v) + case ssaop.OpIfElseUint32s: + return rewriteValue_OpIfElseUint32s(v) + case ssaop.OpIfElseUint64s: + return rewriteValue_OpIfElseUint64s(v) + case ssaop.OpIfElseUint8s: + return rewriteValue_OpIfElseUint8s(v) case ssaop.OpInterCall: v.Op = ssaop.OpARM64CALLinter return true @@ -19812,6 +19840,842 @@ func rewriteValue_OpARM64XORshiftRO(v *ssa.Value) bool { } return false } +func rewriteValue_OpARM64ZSELB(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELB (ZADDB x y) x mask) + // result: (ZADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZADDB x y) y mask) + // result: (ZADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZADDB x y) z mask) + // result: (ZADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELB (ZSQADDB x y) x mask) + // result: (ZSQADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZSQADDB x y) y mask) + // result: (ZSQADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZSQADDB x y) z mask) + // result: (ZSQADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELB (ZUQADDB x y) x mask) + // result: (ZUQADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZUQADDB x y) y mask) + // result: (ZUQADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZUQADDB x y) z mask) + // result: (ZUQADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELD (ZADDD x y) x mask) + // result: (ZADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZADDD x y) y mask) + // result: (ZADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZADDD x y) z mask) + // result: (ZADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZFADDD x y) x mask) + // result: (ZFADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZFADDD x y) y mask) + // result: (ZFADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZFADDD x y) z mask) + // result: (ZFADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZSQADDD x y) x mask) + // result: (ZSQADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) y mask) + // result: (ZSQADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) z mask) + // result: (ZSQADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZUQADDD x y) x mask) + // result: (ZUQADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZUQADDD x y) y mask) + // result: (ZUQADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZUQADDD x y) z mask) + // result: (ZUQADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELH(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELH (ZADDH x y) x mask) + // result: (ZADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZADDH x y) y mask) + // result: (ZADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZADDH x y) z mask) + // result: (ZADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELH (ZSQADDH x y) x mask) + // result: (ZSQADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZSQADDH x y) y mask) + // result: (ZSQADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZSQADDH x y) z mask) + // result: (ZSQADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELH (ZUQADDH x y) x mask) + // result: (ZUQADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZUQADDH x y) y mask) + // result: (ZUQADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZUQADDH x y) z mask) + // result: (ZUQADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELS (ZADDS x y) x mask) + // result: (ZADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZADDS x y) y mask) + // result: (ZADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZADDS x y) z mask) + // result: (ZADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZFADDS x y) x mask) + // result: (ZFADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZFADDS x y) y mask) + // result: (ZFADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZFADDS x y) z mask) + // result: (ZFADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZSQADDS x y) x mask) + // result: (ZSQADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZSQADDS x y) y mask) + // result: (ZSQADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZSQADDS x y) z mask) + // result: (ZSQADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZUQADDS x y) x mask) + // result: (ZUQADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZUQADDS x y) y mask) + // result: (ZUQADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZUQADDS x y) z mask) + // result: (ZUQADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + return false +} func rewriteValue_OpAddr(v *ssa.Value) bool { v_0 := v.Args[0] // match: (Addr {sym} base) @@ -20546,6 +21410,156 @@ func rewriteValue_OpHmul32u(v *ssa.Value) bool { return true } } +func rewriteValue_OpIfElseFloat32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseFloat32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseFloat64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseFloat64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt16s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt16s x mask y) + // result: (ZSELH x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELH) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt8s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt8s x mask y) + // result: (ZSELB x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELB) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint16s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint16s x mask y) + // result: (ZSELH x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELH) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint8s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint8s x mask y) + // result: (ZSELB x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELB) + v.AddArg3(x, y, mask) + return true + } +} func rewriteValue_OpIsInBounds(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/simd/archsimd/_gen/simdgen/arch.go b/src/simd/archsimd/_gen/simdgen/arch.go index 608805f4c6d446..cd6d7284620252 100644 --- a/src/simd/archsimd/_gen/simdgen/arch.go +++ b/src/simd/archsimd/_gen/simdgen/arch.go @@ -164,18 +164,26 @@ var sveArrangements = []string{"B", "H", "S", "D"} // predicated ops are supported. The names are the parameters of the generated // simdARM64SVEOps function, bound to concrete regInfo values in ARM64Ops.go. var sveRegInfoKeys = []string{ - "z11", // 1 Z in, 1 Z out (unary, e.g. NEG) - "z21", // 2 Z in, 1 Z out (binary, e.g. unpredicated ADD) - "z2kk", // 2 Z in, 1 P (governing predicate) in, 1 P out (predicated compare, e.g. ZCMPGT) + "z11", // 1 Z in, 1 Z out (unary, e.g. NEG) + "z21", // 2 Z in, 1 Z out (binary, e.g. unpredicated ADD) + "z2kk", // 2 Z in, 1 P (governing predicate) in, 1 P out (predicated compare, e.g. ZCMPGT) + "z2kv", // 2 Z in, 1 P (select predicate) in, 1 Z out (constructive, e.g. ZSEL) + "z2kvPred", // 2 Z in, 1 P (governing predicate) in, 1 Z out (destructive, e.g. ZADD/M) + // 3 Z in, 1 P (governing predicate) in, 1 Z out, destination shared with the + // first input: a destructive predicated op behind a MOVPRFX, e.g. ZADDMergingPrefixed. + "z3kvPredResultInArg0", } var sveRegInfoSet = map[string]bool{ - "z11": true, - "z21": true, - "z2kk": true, + "z11": true, + "z21": true, + "z2kk": true, + "z2kv": true, + "z2kvPred": true, + "z3kvPred": true, } -const sveRegInfoParams = "z11, z21, z2kk regInfo" +const sveRegInfoParams = "z11, z21, z2kk, z2kv, z2kvPred, z3kvPred regInfo" const sveGeneratedHeader = `// Code generated by 'simdgen -o godefs -goroot $GOROOT -arch sve -arm64Path $ARM64_ISA_PATH go_sve.yaml types.yaml categories.yaml'; DO NOT EDIT. ` diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go index 246d969cd4bd94..65b85612ffb1b7 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go @@ -163,6 +163,24 @@ func writeSIMDMachineOps(buffer *bytes.Buffer, ops []Operation) { if shapeOut == OneVregOutAtIn { resultInArg0 = true } + if CurrentArch().isSVE() { + switch idx := gOp.sveInPlaceInput(); { + case idx < 0: + // Constructive: the destination is independent of the sources. + case idx == 0: + // The instruction overwrites its first source. A commutative one is + // left unconstrained — the ssa-to-prog helper puts the destination in + // place by swapping the operands or by prefixing a MOVPRFX. A + // non-commutative one cannot be fixed by swapping, so pin the + // destination to the first source instead. + if !gOp.Commutative { + resultInArg0 = true + } + default: + panic(fmt.Errorf("simdgen: %s overwrites input %d; only the first input is supported: %s", + gOp.Asm, idx, gOp)) + } + } var memOpData *opData regInfoMerging := regInfo hasMerging := false @@ -214,6 +232,44 @@ func writeSIMDMachineOps(buffer *bytes.Buffer, ops []Operation) { } } else { opsData = append(opsData, opData{asm, gOp.Asm, len(gOp.In), regInfo, gOp.Commutative, outType, resultInArg0}) + // The inVariant implies machine ops only: one predicated instruction + // per governing-predicate qualifier the encoding supports, reached by + // peephole rather than by any API of its own. + for _, pred := range gOp.svePredicatedOps() { + predRegInfo, err := makeRegInfo(pred, NoMem) + if err != nil { + panic(err) + } + predResultInArg0 := false + switch idx := pred.sveInPlaceInput(); { + case idx < 0: + case idx == 0: + // Where the first input is the merge source it is the whole + // reason the destination is pinned, so commutativity — which + // is about the two sources — does not enter into it. + predResultInArg0 = pred.sveMergeSourceIn0 || !pred.Commutative + default: + panic(fmt.Errorf("simdgen: %s overwrites input %d; only the first input is supported: %s", + pred.Asm, idx, pred)) + } + opsData = append(opsData, opData{machineOpName(OneMask, pred), pred.Asm, len(pred.In), + predRegInfo, pred.Commutative, outType, predResultInArg0}) + // There is no zeroing machine op here for Masked to fold into: + // every ARM64 instruction that has both an unpredicated and a + // predicated encoding is /M-only. The /Z forms belong to + // predicated-only instructions (ABS, NEG, NOT, ...), where + // sveImplicitPredPeepholes folds Masked into them. + if prefixed := pred.sveMergingPrefixedOp(); prefixed != nil { + prefixedRegInfo, err := makeRegInfo(*prefixed, NoMem) + if err != nil { + panic(err) + } + // The extra input is the value the destination starts out + // holding, so the destination must share its register. + opsData = append(opsData, opData{machineOpName(OneMask, *prefixed), prefixed.Asm, len(prefixed.In), + prefixedRegInfo, false, outType, true}) + } + } if memOpData != nil { if *op.MemFeatures != "vbcst" { panic("simdgen only knows vbcst for mem ops for now") diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go index 9353fbb6dfcc56..6f72183dfeff62 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go @@ -369,6 +369,23 @@ func (m {{.Name}}) Store(bits []uint16) { func (m {{.Name}}) store(bits []uint16) {{end}} +{{define "sveIfElseTmpl"}} +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x {{.Name}}) IfElse(mask Mask{{.ElemBits}}s, y {{.Name}}) {{.Name}} + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x {{.Name}}) Masked(mask Mask{{.ElemBits}}s) {{.Name}} { + var zero {{.Name}} + return x.IfElse(mask, zero) +} +{{end}} + {{define "sveStringTmpl"}} {{- if eq .Type "mask"}} // String returns a string representation of SIMD mask m: 1 for an active lane, @@ -944,9 +961,13 @@ type psve struct { } } } - // Scalable types print only the lanes that exist at the runtime vector - // length, so their String is generated here rather than by tmplgen (which - // generates the fixed-width ones from a constant lane count). + // TODO: these type utility methods can also be generated by tmplgen, or we can move other arches from + // tmplgen to here. + if typeDef.IsScalable() && typeDef.Type() != "mask" { + if err := t.ExecuteTemplate(buffer, "sveIfElseTmpl", typeDef); err != nil { + panic(fmt.Errorf("failed to execute sveIfElseTmpl template for type %s: %w", typeDef.Name(), err)) + } + } if typeDef.IsScalable() { if err := t.ExecuteTemplate(buffer, "sveStringTmpl", typeDef); err != nil { panic(fmt.Errorf("failed to execute sveStringTmpl template for type %s: %w", typeDef.Name(), err)) diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go index a1c0ecb552f9e2..1eebd1d2afd7dc 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go @@ -198,7 +198,7 @@ func expandFormatSpecifiers(s string, elemBits int) string { // the predicated machine op with a synthesized all-true predicate. // // The all-true predicate is PWHILELT(0, lanes) with lanes = -// maxVectorBits/elemBits, the lane count at the maximum supported vector length. +// MaxVectorBits/elemBits, the lane count at the maximum supported vector length. // Since PWHILELT saturates // (lane i is set while i < hi), this predicate is all-true at any smaller VL too, // so it stands in for the not-yet-available PTRUE. @@ -214,6 +214,93 @@ func sveImplicitPredRule(gOp Operation, asm, args string) string { gOp.GenericName(), args, asm, args, letter, lanes) } +// sveAllTruePattern returns the rule text for the synthesized all-true +// governing predicate of an operation, the same term sveImplicitPredRule +// produces. Matching it, rather than a wildcard, is what makes the peepholes +// below sound: they replace the predicate, so they may only fire on one that +// selects every lane. +func sveAllTruePattern(gOp Operation) string { + elemBits := *gOp.Out[0].ElemBits + return fmt.Sprintf("(Select0 (PWHILELT%s (MOVDconst [0]) (MOVDconst [%d])))", + sveArrangementLetter(gOp), types.MaxVectorBits/elemBits) +} + +// sveImplicitPredPeepholes returns the rules that fold a select over an +// operation whose governing predicate is implicit-all-true into that +// operation's genuinely predicated forms. The operation computes every lane and +// the select then throws most of them away, so the predicate the select +// describes can simply take the place of the all-true one: +// +// (ZSELB (ZABSB x ) z mask) => (ZABSMergingB z x mask) +// +// The select's "else" operand stays in the inactive lanes, which is what merging +// predication does; naming that operand is something a constructive instruction +// does natively, so no MOVPRFX is involved. Masked -- a select against zero -- +// folds through the same rule, with the zero vector as the else operand. +func sveImplicitPredPeepholes(gOp Operation, asm, args string) string { + if governingInput(gOp.In) < 0 { + return "" + } + sel := "ZSEL" + sveArrangementLetter(gOp) + var rules string + for _, pred := range gOp.svePredicatedOps() { + if sveMaskSuffix(pred) != "Merging" || !pred.sveMergeSourceIn0 { + continue + } + rules += fmt.Sprintf("(%s (%s %s %s) z mask) => (%s z %s mask)\n", + sel, asm, args, sveAllTruePattern(gOp), machineOpName(OneMask, pred), args) + } + return rules +} + +// sveMergingPeephole returns the rules that fold a select over an unpredicated +// SVE operation into the operation's merging-predicated form, e.g. +// +// (ZSELB (ZADDB x y) x mask) => (ZADDMergingB x y mask) +// +// which is what x.Add(y).IfElse(mask, x) lowers to. Merging predication keeps +// the destination — and an SVE predicated instruction is destructive, so the +// destination is the first source — which is why the select's "else" operand +// must be that same operand for this shortest form to hold. A commutative +// operation gets the mirrored rule too, since either source can play that role. +// +// A commutative operation also gets the general rule, whose "else" operand is +// unrestricted because the lowering prefixes a MOVPRFX: +// +// (ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask) +// +// It is emitted last so the two rules above win where they apply and save the +// prefix. See [Operation.sveMergingPrefixedOp] for why it needs commutativity. +func sveMergingPeephole(gOp Operation) string { + var pred *types.Operand + vregs := 0 + for i := range gOp.In { + switch { + case gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && !gOp.In[i].IsGoverning(): + pred = &gOp.In[i] + case gOp.In[i].Class == "vreg": + vregs++ + } + } + if pred == nil || *pred.Predication != "M" || vregs != 2 { + return "" + } + sel := "ZSEL" + sveArrangementLetter(gOp) + unpred := machineOpName(NoMask, gOp) + merging := machineOpName(OneMask, gOp) + rules := fmt.Sprintf("(%s (%s x y) x mask) => (%s x y mask)\n", sel, unpred, merging) + if gOp.Commutative { + rules += fmt.Sprintf("(%s (%s x y) y mask) => (%s y x mask)\n", sel, unpred, merging) + } + if gOp.sveMergingPrefixedOp() != nil { + // The general case: any "else" operand, reached by prefixing a MOVPRFX. + // It subsumes the two rules above, which come first so that a select + // whose "else" is already one of the sources keeps the shorter encoding. + rules += fmt.Sprintf("(%s (%s x y) z mask) => (%s z x y mask)\n", sel, unpred, machineOpName(OneMask, *gOp.sveMergingPrefixedOp())) + } + return rules +} + // writeSIMDRules generates the lowering and rewrite rules for ssa and writes it to simdAMD64.rules // within the specified directory. func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { @@ -275,6 +362,9 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { // mask-conversion machinery below (SVE predicates are represented as-is). if opr.implicitPredCount() > 0 { sveRules = append(sveRules, sveImplicitPredRule(gOp, asm, data.Args)) + if r := sveImplicitPredPeepholes(gOp, asm, data.Args); r != "" { + sveRules = append(sveRules, r) + } asmCheck[asm] = true continue } @@ -498,6 +588,11 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { data.ArgsOut = "..." } data.TplName = tplName + for _, pred := range gOp.svePredicatedOps() { + if r := sveMergingPeephole(pred); r != "" { + sveRules = append(sveRules, r) + } + } if opr.NoGenericOps != nil && *opr.NoGenericOps == "true" || opr.SkipMaskedMethod() { optData = append(optData, data) @@ -524,7 +619,10 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { } } + // Signed and unsigned element types share machine ops, so the same rule can + // be produced more than once. slices.Sort(sveRules) + sveRules = slices.Compact(sveRules) for _, rule := range sveRules { buffer.WriteString(rule) } diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go index 6dbf5a43094537..c128bc0cc2b422 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go @@ -191,6 +191,19 @@ func writeSIMDSSA(buffer *bytes.Buffer, ops []Operation) { registerRegShape(regShape, caseStr, op) return nil } + // An SVE inVariant implies machine ops that are not operations of their own, + // so expand them here too; each needs its own ssa-to-prog case. + expanded := make([]Operation, 0, len(ops)) + for _, op := range ops { + expanded = append(expanded, op) + for _, pred := range op.svePredicatedOps() { + expanded = append(expanded, pred) + if prefixed := pred.sveMergingPrefixedOp(); prefixed != nil { + expanded = append(expanded, *prefixed) + } + } + } + ops = expanded for _, op := range ops { shapeIn, shapeOut, maskType, immType, gOp, immOpArg := op.shape() asm := machineOpName(maskType, gOp) @@ -201,7 +214,11 @@ func writeSIMDSSA(buffer *bytes.Buffer, ops []Operation) { caseStr := fmt.Sprintf("ssaop.Op%s%s", archInfo.ArchUpper, asm) isZeroMasking := false if shapeIn == OneKmaskIn || shapeIn == OneKmaskImmIn { - if gOp.Zeroing == nil || *gOp.Zeroing { + if (gOp.Zeroing == nil || *gOp.Zeroing) && !CurrentArch().isSVE() { + // x86 spells the zeroing/merging choice as an assembler suffix on + // the masked instruction. SVE encodes it in the governing predicate + // operand itself (Pg/Z or Pg/M), which the ssa-to-prog helper + // already emits, so there is no suffix to parse. ZeroingMask = append(ZeroingMask, caseStr) isZeroMasking = true } diff --git a/src/simd/archsimd/_gen/simdgen/gen_utility.go b/src/simd/archsimd/_gen/simdgen/gen_utility.go index bd78c71b6ee7fb..44f2ba629cc1b0 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_utility.go +++ b/src/simd/archsimd/_gen/simdgen/gen_utility.go @@ -165,7 +165,7 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS hasVreg := false hasListIn := false for _, in := range op.In { - if in.IsImplicitAllTrue() { + if in.IsGoverning() { // An SVE implicit-all-true governing predicate is not part of the Go // API: it must not count as a mask input here, so the op classifies as // an unpredicated (PureVregIn/NoMask) op. The machine op and lowering @@ -194,8 +194,14 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS if immAsmPos == outputReg { immOpIdx += "Out" } - } else if in.Class == "mask" { + } else if in.Class == "mask" && (!CurrentArch().isSVE() || in.Predication != nil) { maskCount++ + } else if in.Class == "mask" { + // An SVE predicate operand with no /M or /Z qualifier is , a plain + // data operand (SEL's select predicate) rather than a governing + // predicate: the operation is unpredicated and the mask is just an + // argument. regShape still counts it as a predicate register. + hasVreg = true } else { if immAsmPos == in.AsmPos { immOpIdx += fmt.Sprintf("In%d", in.AsmPos) @@ -356,6 +362,20 @@ func (op *Operation) regShape(mem memShape) (string, error) { panic("simdgen does not understand memory as output as of now") } regInfo += fixedName + if CurrentArch().isSVE() { + // A governing predicate supplied by the caller (/M or /Z, from the paired + // predicated encoding) means the instruction is predicated and + // destructive; a plain predicate operand (SEL's ) is not, and an + // implicit-all-true predicate is synthesized rather than passed in. They + // share register classes but need different ssa-to-prog helpers, so give + // the caller-predicated form its own shape name. + for i := range gOp.In { + if gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && !gOp.In[i].IsGoverning() { + regInfo += "Pred" + break + } + } + } if CurrentArch().isSVE() && strings.HasPrefix(regInfo, "v") { // SVE vectors live in the scalable Z bank, not the NEON V bank, so name // their shapes with a "z" (z21, z11, ...). This keeps the generated @@ -941,6 +961,13 @@ func (o *Operation) hasMaskedMerging(maskType maskShape, outType outShape) bool } } } + if CurrentArch().isSVE() { + // AMD64 merging takes the merge source as an extra destination operand. + // An SVE predicated instruction is destructive — it merges into its own + // first source — so there is no separate operand to add, and the merging + // form is just the /M-predicated machine op. + return false + } // BLEND and VMOVDQU are not user-facing ops so we should filter them out. return o.OperandOrder == nil && maskType == OneMask && outType == OneVregOut && len(o.InVariant) == 1 && !strings.Contains(o.Asm, "BLEND") && !strings.Contains(o.Asm, "VMOVDQU") diff --git a/src/simd/archsimd/_gen/simdgen/godefs.go b/src/simd/archsimd/_gen/simdgen/godefs.go index fcdb1ee0c66401..c1c481cc60c60a 100644 --- a/src/simd/archsimd/_gen/simdgen/godefs.go +++ b/src/simd/archsimd/_gen/simdgen/godefs.go @@ -44,6 +44,19 @@ type Operation struct { // // For masked operations, this will have the mask operand appended. In []types.Operand + + // sveMergingPrefixed marks the MOVPRFX-prefixed variant of a merging + // predicated operation, built by [Operation.sveMergingPrefixedOp]. It exists + // only to give that variant a machine-op name of its own. + sveMergingPrefixed bool + + // sveMergeSourceIn0 marks a merging predicated operation whose first input + // is the value the destination starts out holding, and which therefore has + // to share that input's register. Merging predication leaves the inactive + // lanes of the destination alone, so that value is an operand of the + // operation whether the instruction names it (a constructive one does, as + // ABS , /M, ) or a MOVPRFX has to put it there. + sveMergeSourceIn0 bool } func (o *Operation) IsMasked() bool { @@ -106,6 +119,13 @@ func (o *Operation) DecodeUnified(v *unify.Value) error { } isMasked := o.IsMasked() + if CurrentArch().isSVE() { + // An SVE inVariant is the operation's predicated encoding, not a separate + // masked API. The operation keeps its unpredicated name and inputs; the + // predicate is picked up later, by the machine op and peephole generators, + // through svePredicated. + isMasked = false + } // Compute full Go method name. o.Go = o.rawOperation.Go @@ -133,7 +153,10 @@ func (o *Operation) DecodeUnified(v *unify.Value) error { o.Documentation += "\n" + reForName.ReplaceAllString(*o.rawOperation.AddDoc, o.Go) } - o.In = append(o.rawOperation.In, o.rawOperation.InVariant...) + o.In = o.rawOperation.In + if !CurrentArch().isSVE() { + o.In = append(o.rawOperation.In, o.rawOperation.InVariant...) + } // For operations that read only the lower half of input registers (indicated by hiHalfAsm), // add a doc note showing the compositional pattern for the upper half. @@ -205,6 +228,19 @@ var demotingConvertOps = map[string]bool{ "VPMOVWBMasked128": true, "VPMOVSWBMasked128": true, "VPMOVUSWBMasked128": true, } +// sveMaskSuffix returns the machine-op name suffix for a masked operation: +// "Merging" for an SVE /M predicate, "Masked" for /Z and for every other target. +func sveMaskSuffix(gOp Operation) string { + if CurrentArch().isSVE() { + for i := range gOp.In { + if gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && *gOp.In[i].Predication == "M" { + return "Merging" + } + } + } + return "Masked" +} + // sveArrangementLetter returns the SVE element-size arrangement letter // (B=8, H=16, S=32, D=64) that names an SVE machine op, or "" when the target // is not SVE. The letter comes from the operation's governing element width: @@ -243,7 +279,15 @@ func sveArrangementLetter(gOp Operation) string { func machineOpName(maskType maskShape, gOp Operation) string { asm := gOp.Asm if maskType == OneMask { - asm += "Masked" + // An SVE predicated encoding is either merging (/M) or zeroing (/Z), and + // an operation may offer only one of them; name the machine op after the + // qualifier so both can coexist and so the peepholes can tell which + // (IfElse folds into merging, Masked into zeroing). Elsewhere a mask is + // always zeroing, and keeps the historical "Masked" name. + asm += sveMaskSuffix(gOp) + if gOp.sveMergingPrefixed { + asm += "Prefixed" + } } // For ARM64, use arrangement to create distinct SSA op names if letter := sveArrangementLetter(gOp); letter != "" { @@ -361,6 +405,161 @@ func compareOperands(x, y *types.Operand) int { } } +// isInPlaceRegName reports whether an ARM register symbol names an operand that +// is written in place: , and friends, as opposed to or . +func isInPlaceRegName(name string) bool { + return len(name) >= 3 && name[1] == 'd' +} + +// sveInPlaceInput returns the index in op.In of the input naming the same +// register as the destination — the operand a destructive instruction +// overwrites — or -1 when the instruction is constructive. +// +// It fails loudly on a destination that is written in place but is not among +// the inputs, e.g. the accumulator of MLA , /M, , : that needs +// a machine op with an extra input, which simdgen does not build yet, and +// silently treating it as constructive would generate wrong code. +func (op Operation) sveInPlaceInput() int { + if len(op.Out) != 1 || op.Out[0].RegName == nil { + return -1 + } + dst := *op.Out[0].RegName + for i := range op.In { + if op.In[i].RegName != nil && *op.In[i].RegName == dst { + return i + } + } + if isInPlaceRegName(dst) { + panic(fmt.Errorf("simdgen: %s writes %s in place but does not read it as an input; "+ + "this shape is not supported yet: %s", op.Asm, dst, op)) + } + return -1 +} + +// svePredicatedOps returns the machine-level operations implied by the +// operation's inVariant: the same operation with the governing predicate as an +// ordinary input, once per qualifier the encoding supports. The inVariant +// implies machine ops only — the API is generated from the unpredicated in/out +// — and these are what the Masked/IfElse peepholes fold into. +func (op Operation) svePredicatedOps() []Operation { + if !CurrentArch().isSVE() || len(op.InVariant) != 1 || op.InVariant[0].Predication == nil { + return nil + } + var out []Operation + for i, predicate := range op.InVariant { + if predicate.Predication == nil { + continue + } + for _, qual := range *predicate.Predication { + // "M" (merging), "Z" (zeroing), or both: an encoding that offers each + // gets a machine op for each, and only the peepholes that apply to it. + q := string(qual) + p := predicate + p.Predication = &q + pred := op + // Give every operand the symbol it has in this encoding, so the + // operation describes the instruction that will be emitted and its + // shape can be read off it the same way as an unpredicated one. + pred.In = withPredRegNames(op.In, i) + pred.Out = withPredRegNames(op.Out, i) + // An operation with an unpredicated encoding has no governing + // predicate to begin with, so the variant's is a new input. One + // without (ABS) already carries its own, hidden behind an all-true + // predicate; the variant supplies the real one in its place, rather + // than a second one. + if idx := governingInput(pred.In); idx >= 0 { + pred.In[idx] = p + } else { + pred.In = append(pred.In, p) + } + pred.InVariant = nil + pred.sortOperand() + if q == "M" && pred.sveInPlaceInput() < 0 { + // A constructive instruction names its destination separately + // from its sources, and merging predication preserves that + // destination's inactive lanes, so the value it starts out + // holding is a real operand. Without it the machine op would + // claim to write a register it in fact only partly writes. + merge := pred.Out[0] + pred.In = append([]types.Operand{merge}, pred.In...) + pred.sveMergeSourceIn0 = true + } + out = append(out, pred) + } + } + return out +} + +// sveMergingPrefixedOp returns the MOVPRFX-prefixed variant of a merging +// predicated operation, or nil when the operation cannot use one. +// +// A merging SVE instruction is destructive — it merges into its own first +// source — so on its own it can only express a select whose "else" operand is +// that same source. Prefixing MOVPRFX lifts that: given +// +// ZMOVPRFX Zx, Pg/M, Zd +// ZADD Zy, Zd, Pg/M, Zd +// +// the destination holds x+y on the active lanes and whatever it already held on +// the inactive ones, so the "else" operand can be any value. The returned +// operation carries that value as an extra leading input, which makes it the +// operand the destination must share a register with (resultInArg0) and gives +// the operation a three-vreg register shape of its own. +// +// It is offered only for a commutative operation. The prefixed instruction must +// not name the destination in any operand position other than the destructive +// one, i.e. the ZADD above needs Zy != Zd; a commutative operation can always +// satisfy that by swapping its two sources, and a non-commutative one cannot. +func (op Operation) sveMergingPrefixedOp() *Operation { + if !CurrentArch().isSVE() || !op.Commutative || op.sveInPlaceInput() != 0 { + return nil + } + if len(op.Out) != 1 || op.Out[0].RegName == nil { + return nil + } + if sveMaskSuffix(op) != "Merging" { + return nil + } + // The extra input is the destination read before the operation, so it takes + // the destination's symbol; the source it displaces becomes the MOVPRFX's + // Zn, which is the symbol that instruction gives it. + merge := op.Out[0] + prefixed := op + prefixed.In = make([]types.Operand, 0, len(op.In)+1) + prefixed.In = append(prefixed.In, merge) + prefixed.In = append(prefixed.In, op.In...) + movprfxSrc := "Zn" + prefixed.In[1].RegName = &movprfxSrc + prefixed.sveMergingPrefixed = true + prefixed.sveMergeSourceIn0 = true + return &prefixed +} + +// governingInput returns the index of the governing predicate in ops, or -1 +// when there is none. +func governingInput(ops []types.Operand) int { + for i := range ops { + if ops[i].IsGoverning() { + return i + } + } + return -1 +} + +// withPredRegNames copies operands with each one's register symbol replaced by +// the symbol it has in predicated encoding i, where it has one. +func withPredRegNames(ops []types.Operand, i int) []types.Operand { + out := make([]types.Operand, len(ops)) + copy(out, ops) + for j := range out { + if names := out[j].PredRegName; names != nil && i < len(*names) { + name := (*names)[i] + out[j].RegName = &name + } + } + return out +} + // implicitPredCount reports whether the op has an implicit-all-true governing // predicate input, as a count (0 or 1). An instruction has at most one governing // predicate — the single mask input carrying a /Z or /M qualifier (see the @@ -372,7 +571,7 @@ func compareOperands(x, y *types.Operand) int { func (op Operation) implicitPredCount() int { n := 0 for i := range op.In { - if op.In[i].IsImplicitAllTrue() { + if op.In[i].IsGoverning() { n++ } } diff --git a/src/simd/archsimd/_gen/simdgen/sve/emit.go b/src/simd/archsimd/_gen/simdgen/sve/emit.go index 877667c755f779..5c0fa294819583 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/emit.go +++ b/src/simd/archsimd/_gen/simdgen/sve/emit.go @@ -7,6 +7,7 @@ package sve import ( "cmp" "fmt" + "log" "slices" "strings" @@ -37,6 +38,10 @@ func asComment(text string, width int) string { return strings.Join(lines, "\n") } +// mixedWidthLogged dedupes the mixed-element-width warning by mnemonic, so a +// conversion family with many encodings logs once per generate run. +var mixedWidthLogged = map[string]bool{} + // emit renders an operand as a unify value. Z-vectors and predicates are // scalable (a base type and per-operand element width, no fixed bits/lanes); // mem, immediate and special operands are opaque (class and position only). @@ -68,31 +73,54 @@ func (op *Operand) emit() *unify.Value { // instructions support only one; this records which. db.Add("predication", unify.NewValue(unify.NewStringExact(op.Predication))) } - if op.role == "mask" { - // role "mask" is precisely the governing predicate: the operand named - // (buildOperandList assigns the role; every instruction has at most one). It - // is implicit-all-true — dropped from the unpredicated Go API and - // synthesized as an all-true predicate at lowering, so predicated-only - // instructions (e.g. ZCMPGT) expose an unpredicated API. Flagging it here, - // not in the user's go_*.yaml, keeps the YAML unpredicated. - // - // The governing predicate is identified by name, not by a /Z or /M - // qualifier: most data-processing ops write /Z or /M, but some - // governing predicates have no qualifier (e.g. the store ST1B {.B}, - // , [...]). Either way it is . Source predicates / (e.g. in - // AND .B, /Z, .B, .B) are ordinary numbered inputs (role - // "opN"), real data, and are never flagged all-true. - db.Add("implicitAllTrue", unify.NewValue(unify.NewStringExact("true"))) + if op.governing { + // This operand is a governing predicate. + db.Add("governing", unify.NewValue(unify.NewStringExact("true"))) } if op.isList { // This register came from a single-register list ("{ . }"), a // distinct assembler encoding from a bare register. db.Add("listNumber", unify.NewValue(unify.NewStringExact("0"))) } + if op.regName != "" { + // The assembly template's register symbol, e.g. "Zdn", "Zn", "Pg". + db.Add("regName", unify.NewValue(unify.NewStringExact(op.regName))) + } + // The symbol this operand has in each predicated encoding, indexed to + // match the def's inVariant. The symbols can differ from the unpredicated + // ones to predicated ones: + // ADD , , unpredicated + // ADD , /M, , predicated + // + // [groupPredicationForms] folds the two into one def. + // simdgen needs these symbols to recognize resultInArg0. + names := make([]*unify.Value, len(op.predRegName)) + for i, n := range op.predRegName { + names[i] = unify.NewValue(unify.NewStringExact(n)) + } + db.Add("predRegName", unify.NewValue(unify.NewTuple(names...))) db.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.AsmPos)))) return unify.NewValue(db.Build()) } +// pickRegNames returns operand idx's symbol in each predicated encoding, in +// variant order. The encodings passed [sameOperandShape], so idx addresses the +// matching operand in every one of them. +func pickRegNames(variants []predVariant, idx int, sel func(predVariant) []string) []string { + if len(variants) == 0 { + return nil + } + out := make([]string, len(variants)) + for i, pv := range variants { + names := sel(pv) + if idx >= len(names) { + panic(fmt.Sprintf("operand %d has no counterpart in predicated encoding %d", idx, i)) + } + out[i] = names[idx] + } + return out +} + // emitOne emits a single instruction def from a fully-instantiated operand list: // the destination is the output, every other operand (including a governing // predicate) is a literal input. @@ -108,11 +136,26 @@ func (inst *Instruction) emitOne(asm string, ops []Operand) *unify.Value { db.Add("details", unify.NewValue(unify.NewStringExact(asComment(doc, 80)))) } + // One def can describe several encodings of one operation, grouped by + // [groupPredicationForms] or [groupPredicatedOnly], so each operand also + // carries the symbol it has in each predicated encoding. The symbols are + // matched up in template order, so they must be attached before the sort + // below reorders the inputs. var inOps, outOps []Operand + var outIdx, inIdx int for _, op := range ops { - if op.role == "destination" { + switch { + case op.governing: + // The governing predicate is the operand the paired encodings differ in, so + // it is not one of the symbols they are matched up by. + inOps = append(inOps, op) + case op.role == "destination": + op.predRegName = pickRegNames(inst.predVariants, outIdx, func(pv predVariant) []string { return pv.outRegNames }) + outIdx++ outOps = append(outOps, op) - } else { + default: + op.predRegName = pickRegNames(inst.predVariants, inIdx, func(pv predVariant) []string { return pv.inRegNames }) + inIdx++ inOps = append(inOps, op) } } @@ -134,7 +177,17 @@ func (inst *Instruction) emitOne(asm string, ops []Operand) *unify.Value { outs = append(outs, outOps[i].emit()) } db.Add("in", unify.NewValue(unify.NewTuple(ins...))) - db.Add("inVariant", unify.NewValue(unify.NewTuple())) + var inVar []*unify.Value + for _, pv := range inst.predVariants { + // The governing predicate of the paired predicated encoding. + var pdb unify.DefBuilder + pdb.Add("class", unify.NewValue(unify.NewStringExact("mask"))) + pdb.Add("bits", unify.NewValue(unify.NewStringExact("scalable"))) + pdb.Add("predication", unify.NewValue(unify.NewStringExact(pv.quals))) + pdb.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(pv.predAsmPos)))) + inVar = append(inVar, unify.NewValue(pdb.Build())) + } + db.Add("inVariant", unify.NewValue(unify.NewTuple(inVar...))) db.Add("out", unify.NewValue(unify.NewTuple(outs...))) return unify.NewValue(db.Build()) } @@ -232,10 +285,34 @@ func (inst *Instruction) emitVariants(template []Operand) []*unify.Value { for _, pred := range preds { variant := make([]Operand, len(ops)) copy(variant, ops) + elem := 0 + mixedWidths := false for i := range variant { - if variant[i].Class == "mask" && variant[i].role == "mask" { + if variant[i].Class == "vreg" && variant[i].ElemBits > 0 { + if elem == 0 { + elem = variant[i].ElemBits + } else if variant[i].ElemBits != elem { + mixedWidths = true + } + } + } + for i := range variant { + if variant[i].Class != "mask" { + continue + } + if variant[i].governing { variant[i].Predication = pred } + if variant[i].ElemBits == 0 { + // This predicate doesn't come with an arrangement (which is usual). + // Get it from its peer data operand. + if mixedWidths && !mixedWidthLogged[inst.mnemonic()] { + mixedWidthLogged[inst.mnemonic()] = true + log.Printf("sve: %s: operands have mixed element widths; predicate width provisionally %d — derive esize from the pseudocode before generating an API from this def", + inst.mnemonic(), elem) + } + variant[i].ElemBits = elem + } } defs = append(defs, inst.emitOne(asm, variant)) } diff --git a/src/simd/archsimd/_gen/simdgen/sve/instruction.go b/src/simd/archsimd/_gen/simdgen/sve/instruction.go index 3cc310f4fd482c..bfa4bf6c71f651 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/instruction.go +++ b/src/simd/archsimd/_gen/simdgen/sve/instruction.go @@ -65,6 +65,32 @@ type Instruction struct { // If nil, the first iclass is used. iclass *xmlspec.Iclass mnemonicCache string + // predVariants is set on the unpredicated instruction of a + // predicated/unpredicated pair (see [groupPredicationForms]), one entry per + // predicated machine op the pair implies. It is nil for an instruction that + // comes in one form only. + predVariants []predVariant +} + +// predVariant is one predicated encoding of an operation, as seen from its +// unpredicated sibling: the governing-predicate qualifiers it offers ("M", "Z", +// or "MZ" for an encoding written /, which supports either) and its +// register symbols, in the same order as the sibling's own results and +// non-predicate inputs. +// +// One encoding can imply several machine ops — one per qualifier — but they +// share these symbols, because they are the same encoding. A second entry would +// mean a genuinely separate predicated encoding, which no paired operation in +// the ISA has today; the list exists so that such an encoding could be +// described with its own symbols rather than collapsed onto the first one's. +type predVariant struct { + quals string + outRegNames []string + inRegNames []string + // predAsmPos is the assembly position of the encoding's governing + // predicate: 1 on every encoding grouped today, but recorded rather than + // assumed — PTEST, with no destination, governs from position 0. + predAsmPos int } // ic returns the iclass this logical instruction represents, defaulting to the @@ -286,6 +312,27 @@ func (inst *Instruction) findExplanation(link string) *xmlspec.Explanation { return nil } +// symbolIsGoverning reports whether this instruction's explanation for +// register symbol name (e.g. "Pg") describes it as the governing predicate — +// the spec writes "the governing scalable predicate register" for exactly the +// symbols with that role. found reports whether any explanation names the +// symbol at all. This is the authoritative classification; [buildOperandList] +// cross-checks it against the syntactic /qualifier signal. +func (inst *Instruction) symbolIsGoverning(name string) (governing, found bool) { + want := "<" + name + ">" + for i := range inst.Explanations.Explanations { + e := &inst.Explanations.Explanations[i] + if strings.TrimSpace(e.Symbol.Value) != want { + continue + } + found = true + if strings.Contains(strings.ToLower(e.Account.Intro), "governing") { + return true, true + } + } + return false, found +} + // arngRow is one row of an arrangement size table: the encoding value of the // size field and the resulting element width in bits. type arngRow struct { @@ -399,7 +446,17 @@ func (inst *Instruction) allEncodingOperands() [][]Operand { continue } seen[s] = true - if ops := operandsFromTextA(enc.AsmTemplate.TextA); len(ops) > 0 { + ops := func() []Operand { + // A classification panic names only the operand; add which + // instruction and template it came from. + defer func() { + if r := recover(); r != nil { + panic(fmt.Sprintf("%v\n in %q template %q", r, inst.Title, s)) + } + }() + return operandsFromTextA(enc.AsmTemplate.TextA, inst.symbolIsGoverning) + }() + if len(ops) > 0 { inst.fixMemoryDirection(ops) out = append(out, ops) } @@ -454,7 +511,7 @@ func hasClass(ops []Operand, class string) bool { // bit, or a single no-op pass when the template has no governing predicate. func predicationVariants(ops []Operand) []string { for i := range ops { - if ops[i].Class == "mask" && ops[i].role == "mask" { + if ops[i].governing { if ops[i].Predication == "MZ" { return []string{"M", "Z"} } @@ -464,6 +521,44 @@ func predicationVariants(ops []Operand) []string { return []string{""} } +// predicationForm reports whether this encoding is the predicated or the +// unpredicated form of an operation, as "predicated" / "unpredicated". +// +// It reads the encoding rather than the title: an encoding that takes a +// governing predicate is the predicated one. SVE does also spell this out in +// the title of an operation that has both forms ("ADD (vectors, predicated)" +// and "ADD (vectors, unpredicated)"), and [predicationGroupKey] uses that to pair +// them, but an operation that only comes predicated says nothing in its title — +// both of ABS's encodings are titled plain "ABS". +func (inst *Instruction) predicationForm() string { + for _, ops := range inst.allEncodingOperands() { + for i := range ops { + if ops[i].governing { + return "predicated" + } + } + } + return "unpredicated" +} + +// predicationGroupKey returns the key that groups the encodings of one +// operation: the title with any predicated/unpredicated qualifier removed, e.g. +// both "ADD (vectors, predicated)" and "ADD (vectors, unpredicated)" yield "add +// (vectors)", and both of ABS's encodings yield "abs". +// +// Encodings that are not variations on one another keep distinct titles — "ADD +// (immediate)", "ADD (extended register)" — so they land in groups of their own, +// which groupPredicationForms then leaves alone. +func (inst *Instruction) predicationGroupKey() string { + t := strings.ToLower(inst.Title) + t = strings.ReplaceAll(t, "unpredicated", "") + t = strings.ReplaceAll(t, "predicated", "") + // Tidy the separator the qualifier left behind: "(vectors, )" -> "(vectors)". + t = strings.ReplaceAll(t, ", )", ")") + t = strings.ReplaceAll(t, "( ", "(") + return strings.Join(strings.Fields(t), " ") +} + // documentation returns a one-line description of the instruction. func (inst *Instruction) documentation() string { if len(inst.Desc.Authored.Paragraphs) > 0 { diff --git a/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go b/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go index 530f3d82660bde..9dd7372baae645 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go +++ b/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go @@ -214,9 +214,9 @@ func TestOperandsPredicated(t *testing.T) { ops := parse(t, addPred).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } - want := []string{"vreg:destination", "mask:mask", "vreg:op0", "vreg:op1"} + want := []string{"vreg:destination", "mask:governing", "vreg:op0", "vreg:op1"} if !reflect.DeepEqual(got, want) { t.Errorf("predicated operands = %v, want %v", got, want) } @@ -423,10 +423,10 @@ func TestReductionOutput(t *testing.T) { ops := parse(t, saddv).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } // The scalar result
is a SIMD&FP register destination, not an input. - want := []string{"vreg:destination", "mask:mask", "vreg:op0"} + want := []string{"vreg:destination", "mask:governing", "vreg:op0"} if !reflect.DeepEqual(got, want) { t.Errorf("SADDV operands = %v, want %v", got, want) } @@ -474,13 +474,13 @@ func TestStoreReglist(t *testing.T) { ops := parse(t, st1b).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } // The single-register list unwraps to a vreg (the data source); the memory // operand is the store destination. Order follows the source template. The // predicate is : a governing predicate (role "mask"), even though a // store writes no /Z or /M qualifier. - want := []string{"vreg:op0", "mask:mask", "mem:destination"} + want := []string{"vreg:op0", "mask:governing", "mem:destination"} if !reflect.DeepEqual(got, want) { t.Errorf("ST1B operands = %v, want %v", got, want) } @@ -522,3 +522,119 @@ func TestMemoryOperandClassified(t *testing.T) { } } } + +// opRole renders an operand's partition for test expectations: its role, or +// "governing" for the governing predicate, which has no numbered role. +func opRole(op Operand) string { + if op.governing { + return "governing" + } + return op.role +} + +// absExplanations is the explanation block for the ABS fixtures: the size table +// plus a explanation using the spec's "governing scalable predicate +// register" wording, so parsing exercises the explanation-driven governing +// classification rather than the syntactic fallback. +const absExplanations = ` + + + <T> + + + B + H + S + D +
+
+
+ + <Pg> + Is the name of the governing scalable predicate register, encoded in the "Pg" field. + +
` + +// absSection builds one encoding of ABS, a predicated-only operation: both +// encodings are titled plain "ABS" (nothing in the title says "predicated"), +// and they differ only in the governing predicate's qualifier. +func absSection(id, qual string) string { + return ` + + + + + Absolute value of the signed integer in each active element. + + ABS <Zd>.<T>, <Pg>/` + qual + `, <Zn>.<T> + ` + absExplanations + `` +} + +// TestGroupPredicatedOnly covers the emission path of an operation with no +// unpredicated encoding at all: the merging encoding carries the operation, +// keeping its governing predicate as an implicit-all-true input, and the group +// becomes an inVariant on it; the sibling encoding is covered and not emitted. +func TestGroupPredicatedOnly(t *testing.T) { + m := parse(t, absSection("abs_m", "M")) + z := parse(t, absSection("abs_z", "Z")) + covered := groupPredicationForms([]*Instruction{m, z}) + if covered[m] || !covered[z] { + t.Fatalf("covered[m]=%v covered[z]=%v, want the merging carrier kept and the zeroing sibling covered", covered[m], covered[z]) + } + if len(m.predVariants) != 1 || m.predVariants[0].quals != "M" { + t.Fatalf("carrier predVariants = %+v, want one variant with quals M", m.predVariants) + } + + defs := m.emitAll() + if len(defs) == 0 { + t.Fatal("carrier emitAll returned no defs") + } + for _, d := range defs { + var op struct { + In []struct { + Class string + Predication *string + Governing *bool + RegName *string `unify:"regName"` + PredRegName *[]string `unify:"predRegName"` + } `unify:"in"` + InVariant []struct { + Class string + Predication *string + } `unify:"inVariant"` + Out []struct { + PredRegName *[]string `unify:"predRegName"` + } `unify:"out"` + } + if err := d.Decode(&op); err != nil { + t.Fatal(err) + } + var sawGoverning, sawVreg bool + for _, in := range op.In { + switch in.Class { + case "mask": + if in.Governing == nil || !*in.Governing { + t.Errorf("mask input not marked governing: %+v", in) + } + if in.Predication == nil || *in.Predication != "M" { + t.Errorf("governing predicate predication = %v, want M", in.Predication) + } + sawGoverning = true + case "vreg": + if in.PredRegName == nil || !reflect.DeepEqual(*in.PredRegName, []string{"Zn"}) { + t.Errorf("vreg input predRegName = %v, want [Zn]", in.PredRegName) + } + sawVreg = true + } + } + if !sawGoverning || !sawVreg { + t.Errorf("inputs missing governing mask or vreg: %+v", op.In) + } + if len(op.InVariant) != 1 || op.InVariant[0].Predication == nil || *op.InVariant[0].Predication != "M" { + t.Errorf("inVariant = %+v, want one mask with predication M", op.InVariant) + } + if len(op.Out) != 1 || op.Out[0].PredRegName == nil || !reflect.DeepEqual(*op.Out[0].PredRegName, []string{"Zd"}) { + t.Errorf("out predRegName = %+v, want [Zd]", op.Out) + } + } +} diff --git a/src/simd/archsimd/_gen/simdgen/sve/load.go b/src/simd/archsimd/_gen/simdgen/sve/load.go index 45d367b41af3a7..9b16deaa3be321 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/load.go +++ b/src/simd/archsimd/_gen/simdgen/sve/load.go @@ -5,6 +5,7 @@ package sve import ( + "slices" "sort" "simd/archsimd/_gen/unify" @@ -50,9 +51,197 @@ func Load(path string) ([]*unify.Value, error) { if err != nil { return nil, err } + covered := groupPredicationForms(insts) var defs []*unify.Value for _, inst := range insts { + if covered[inst] { + // The predicated half of a pair; it is emitted as an inVariant of its + // unpredicated sibling so the operation has a single unifier value. + continue + } defs = append(defs, inst.emitAll()...) } return defs, nil } + +// groupPredicationForms pairs the predicated and unpredicated encodings of the +// same operation and folds them into one definition, mirroring how the AMD64 +// loader treats an AVX-512 instruction's optional K-mask: the unpredicated form +// supplies the operation (and therefore the single front-end API), and the +// governing predicate becomes an inVariant that simdgen turns into predicated +// machine ops plus peepholes. +// However, different from AVX-512, where the predication mode is orthogonal to +// the operation as an instruction suffix, SVE's predication modes are separate +// instruction encodings, so the loader has to pair them up. +// +// A pair is only formed when both forms actually exist and their operand shapes +// correspond; the returned set names the predicated instructions that the pair +// covers, which the caller then skips. Everything else — an operation with only +// a predicated form (whose predicate stays implicit-all-true), or only an +// unpredicated one — is emitted unchanged. +func groupPredicationForms(insts []*Instruction) map[*Instruction]bool { + type group struct{ unpred, pred []*Instruction } + groups := map[string]*group{} + for _, inst := range insts { + key := inst.predicationGroupKey() + if key == "" { + continue + } + g := groups[key] + if g == nil { + g = &group{} + groups[key] = g + } + if inst.predicationForm() == "unpredicated" { + g.unpred = append(g.unpred, inst) + } else { + g.pred = append(g.pred, inst) + } + } + + covered := map[*Instruction]bool{} + for _, g := range groups { + if len(g.unpred) == 0 && len(g.pred) > 1 { + groupPredicatedOnly(g.pred, covered) + continue + } + if len(g.unpred) != 1 || len(g.pred) == 0 { + // Not a clean pair (a form is missing, or the title is ambiguous); + // leave both halves to be emitted as they are. + continue + } + un := g.unpred[0] + unOps := un.operands() + var variants []predVariant + for _, pr := range g.pred { + prOps := pr.operands() + if !sameOperandShape(unOps, prOps) { + continue + } + var quals string + for _, q := range predicationVariants(prOps) { + quals += q + } + if quals == "" { + continue + } + // Each encoding carries its own register symbols, so a machine op is + // always generated from the shape of the encoding it comes from. + outs, ins := splitRegNames(prOps) + variants = append(variants, predVariant{quals: quals, outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(prOps)}) + covered[pr] = true + } + if len(variants) == 0 { + continue + } + un.predVariants = variants + } + return covered +} + +// groupPredicatedOnly folds the encodings of an operation that has no +// unpredicated form at all — SVE writes ABS as "ABS ., /M, ." +// and "ABS ., /Z, .", and nothing else. +// +// There is no unpredicated encoding to carry the operation, so one of the +// predicated encodings does. Its governing predicate stays implicit-all-true, so +// the front-end API is still unpredicated, and every qualifier in the group — +// its own included — becomes an inVariant qualifier, which simdgen turns into +// one predicated machine op each for the peepholes to fold into. +// +// Only the merging encoding is used. Zeroing predication on these instructions +// is an Armv9.6-A extension -- ABS assembles to a different opcode under /Z, and +// baseline SVE hardware traps it -- while merging is available wherever SVE is. +// Nothing in what the XML parser exposes tells the two apart: both carry +// instr-class "sve", and the arch_variant element that does record the +// difference is not surfaced. So the zeroing encodings are dropped here rather +// than gated, to be folded in with the rest of SVE2.2 once simdgen can gate on +// the SVE sub-level. +// +// The group is folded only when the encodings are variations on one predication +// mode and nothing else: same operand shape, and one encoding per qualifier. Two +// encodings sharing a qualifier are two different instructions that happen to +// share a title (addressing modes of a load, say), and are left alone. +func groupPredicatedOnly(pred []*Instruction, covered map[*Instruction]bool) { + byQual := map[string]*Instruction{} + shape := pred[0].operands() + for _, inst := range pred { + ops := inst.operands() + if !sameOperandShape(shape, ops) { + return + } + quals := predicationVariants(ops) + if len(quals) != 1 || quals[0] == "" { + return + } + if _, dup := byQual[quals[0]]; dup { + return + } + byQual[quals[0]] = inst + } + base, ok := byQual["M"] + if !ok { + return + } + baseOps := base.operands() + outs, ins := splitRegNames(baseOps) + base.predVariants = []predVariant{{quals: "M", outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(baseOps)}} + for _, inst := range byQual { + if inst != base { + covered[inst] = true + } + } +} + +// governingAsmPos returns the assembly position of the governing predicate in +// ops. Both callers work on encodings already classified as predicated, so a +// missing governing predicate is a broken invariant, not a case. +func governingAsmPos(ops []Operand) int { + for i := range ops { + if ops[i].governing { + return ops[i].AsmPos + } + } + panic("sve: predicated encoding has no governing predicate") +} + +// splitRegNames returns an operand template's register symbols, results first +// and then the non-predicate inputs, in the order sameOperandShape compares +// them, so the two halves of a pair line up element by element. +func splitRegNames(ops []Operand) (outs, ins []string) { + for i := range ops { + if ops[i].governing { + continue + } + if ops[i].role == "destination" { + outs = append(outs, ops[i].regName) + } else { + ins = append(ins, ops[i].regName) + } + } + return outs, ins +} + +// sameOperandShape reports whether two operand templates describe the same +// operation apart from a governing predicate: same result and same sequence of +// non-predicate input classes. The predicated form of a destructive operation +// names its destination twice (once as the in-place source), which +// buildOperandList already turns into a regular input, so the two shapes line up. +func sameOperandShape(a, b []Operand) bool { + split := func(ops []Operand) (outs, ins []string) { + for i := range ops { + if ops[i].governing { + continue // the governing predicate is what differs + } + if ops[i].role == "destination" { + outs = append(outs, ops[i].Class) + } else { + ins = append(ins, ops[i].Class) + } + } + return outs, ins + } + ao, ai := split(a) + bo, bi := split(b) + return slices.Equal(ao, bo) && slices.Equal(ai, bi) +} diff --git a/src/simd/archsimd/_gen/simdgen/sve/operands.go b/src/simd/archsimd/_gen/simdgen/sve/operands.go index 52b2f4b11e1b15..bc905a48ba95da 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/operands.go +++ b/src/simd/archsimd/_gen/simdgen/sve/operands.go @@ -6,6 +6,7 @@ package sve import ( "fmt" + "log" "regexp" "strings" @@ -109,11 +110,16 @@ type Operand struct { // and unknown operands so diagnostics can name what was skipped. Raw string - // role is the operand's internal role: "destination", "op0"/"op1"/..., or - // "mask" (a governing predicate). It drives out/in/inVariant partitioning at - // emit time but is NOT emitted (simdgen orders operands by AsmPos, so a role - // field in the YAML would be redundant). + // role is the operand's internal role: "destination" or "op0"/"op1"/.... + // It drives out/in partitioning at emit time but is NOT emitted (simdgen + // orders operands by AsmPos, so a role field in the YAML would be + // redundant). A governing predicate has no role; it is marked by governing. role string + // governing marks the governing predicate — the operand selecting which + // lanes the instruction acts on, as opposed to a predicate read as data. + // It is classified from the spec's own explanation text for the symbol and + // emitted as the def's "governing" field. + governing bool // arngLink is the link of this operand's arrangement symbol (// // ), used to resolve its per-operand element widths. Empty if the // operand has a fixed or no arrangement. @@ -130,6 +136,11 @@ type Operand struct { isList bool // regName is the inner register symbol, e.g. "Zdn", "Zm", "Pg". regName string + // predRegName is the symbol this operand has in each paired predicated + // encoding, indexed to match the operation's predicated variants (and so the + // inVariant tuple the def carries). It is nil when the operation has no + // predicated form. + predRegName []string } // resultInArg0 reports whether this destination register is also read, i.e. it @@ -215,17 +226,20 @@ type tok struct { } // operandsFromTextA parses operands from an assembly template's / -// sequence, preserving each operand's arrangement-symbol link. -func operandsFromTextA(textA []xmlspec.TextA) []Operand { - return buildOperandList(classifyToks(tokenizeTextA(textA))) +// sequence, preserving each operand's arrangement-symbol link. govern reports +// whether a predicate register symbol is the governing predicate; the real +// loader path passes the instruction's explanation lookup. +func operandsFromTextA(textA []xmlspec.TextA, govern func(regName string) (governing, found bool)) []Operand { + return buildOperandList(classifyToks(tokenizeTextA(textA)), govern) } // operands parses operands from a flattened template string. It cannot recover -// links, so arrangement symbols resolve to empty links; it is used for +// links or explanations, so arrangement symbols resolve to empty links and +// the governing predicate is classified syntactically; it is used for // classification-only paths and tests. The real loader path uses // operandsFromTextA. func operands(asmTemplate string) []Operand { - return buildOperandList(classifyToks(tokenizeString(asmTemplate))) + return buildOperandList(classifyToks(tokenizeString(asmTemplate)), nil) } // tokenizeTextA splits a / sequence into operand tokens on top-level @@ -493,9 +507,9 @@ func isInPlaceReg(name string) bool { // // Unlike an AMD64 AVX-512 K-mask, an SVE governing predicate is NOT optional: // there is no K0-style "no predicate" encoding, so it is a mandatory literal -// input (class "mask", role "mask"), not an inVariant. See the discussion in -// emitOne. -func buildOperandList(parsed []tok) []Operand { +// input (class "mask", marked governing), not an inVariant. See the discussion +// in emitOne. +func buildOperandList(parsed []tok, govern func(regName string) (governing, found bool)) []Operand { var outs, ins []Operand inputCount := 0 destAssigned := false @@ -534,16 +548,49 @@ func buildOperandList(parsed []tok) []Operand { } switch p.operandType { case OperandPReg: - if p.regName == "Pg" || p.predication != "" { - // Governing predicate: the operand named ("g" for governing), a - // mandatory mask input (role "mask", not a numbered opN). Most carry a - // /Z or /M qualifier (predicated data-processing ops), but some do not - // — e.g. the store ST1B {.B}, , [...] governs with a plain - // — so key on the register name, not the qualifier. Source - // predicates / and the destination are ordinary operands, - // filed by place() below. + // The governing predicate is classified from the spec's own words: + // its explanation calls the symbol "the governing scalable predicate + // register". The syntactic signal — the symbol is , or it carries + // a /M or /Z qualifier — is kept as a cross-check, so a shape where + // the two diverge fails loudly instead of misclassifying: the SME + // outer products govern with two predicates spelled / + // (SUMOPA .S, /M, /M, .B, .B), which does not + // fit the one-governing-predicate shape simdgen builds on and must + // be rejected here, not silently halved. The bare-string parse path + // (tests, diagnostics) has no explanations and uses the syntactic + // signal alone. + syntactic := p.regName == "Pg" || p.predication != "" + governing := syntactic + if govern != nil { + if verdict, found := govern(p.regName); found { + governing = verdict + if governing != syntactic { + // The explanation wins, but say so. Two shapes diverge in + // the ISA today, in opposite directions: the MOV alias of + // SEL (MOV ., /M, .), whose keeps + // its "select" description from SEL even though the alias + // writes it with a qualifier; and the bare of the + // predicate-as-counter loads/stores, which the spec calls + // governing though it is neither nor qualified. + // Neither instruction is emitted today, so nothing + // downstream sees the difference — but the second is the + // case the explanation gets right and the syntax cannot. + log.Printf("sve: symbol <%s> (qualifier %q): explanation says governing=%v, syntactic signal says %v; following the explanation", + p.regName, p.predication, governing, syntactic) + } + } + // No explanation for this symbol — an alias template, or a test + // fixture with the explanations stripped — so the syntactic + // signal stands alone. + } + if governing { + // A mandatory mask input, not a numbered opN. Most carry a /Z or + // /M qualifier (predicated data-processing ops), but some do not + // — the store ST1B {.B}, , [...] governs with a plain + // . Source predicates / and the destination are + // ordinary operands, filed by place() below. ins = append(ins, Operand{ - Type: OperandPReg, Class: "mask", role: "mask", + Type: OperandPReg, Class: "mask", governing: true, Predication: p.predication, AsmPos: p.asmPos, arngLink: p.arngLink, fixedElem: p.fixedElem, regName: p.regName, }) @@ -571,6 +618,19 @@ func buildOperandList(parsed []tok) []Operand { }, p.isDestination) } } + // An instruction has at most one governing predicate — everything simdgen + // derives from the field (implicitPredCount, regShape, the all-true + // synthesis) assumes it. The SME outer products break this (SUMOPA governs + // with /M and /M at once) and must be rejected here, not halved. + governCount := 0 + for i := range ins { + if ins[i].governing { + governCount++ + } + } + if governCount > 1 { + panic(fmt.Sprintf("sve: %d governing predicates in one operand list; only one is supported", governCount)) + } return append(outs, ins...) } diff --git a/src/simd/archsimd/_gen/simdgen/types/operation.go b/src/simd/archsimd/_gen/simdgen/types/operation.go index cdadfbc2fa8ae1..35476f7259570f 100644 --- a/src/simd/archsimd/_gen/simdgen/types/operation.go +++ b/src/simd/archsimd/_gen/simdgen/types/operation.go @@ -100,12 +100,38 @@ type Operand struct { // Currently only list number 0 is supported (we might need to teach regalloc handle register lists // to support more than one register in the list). ListNumber *int - // ImplicitAllTrue marks an SVE governing-predicate input that is dropped from - // the user-facing (unpredicated) API: the generated method/generic op/intrinsic - // omit it, and the lowering synthesizes an all-true predicate for it. This is - // how predicated-only SVE instructions (e.g. ZCMPGT) expose an unpredicated Go - // API, for #79781. - ImplicitAllTrue *bool + // RegName is the assembly template's register symbol for this operand, e.g. + // "Zdn", "Zn", "Pg" (SVE only). Comparing it across operands is how the + // shape of an instruction is recognised: an input naming the same register + // as the destination is written in place. + RegName *string + // PredRegName is the symbol this operand has in each of the operation's + // predicated encodings, indexed to match InVariant (SVE only). It is nil + // for an operation with no predicated encoding, and for every other target. + PredRegName *[]string + // Predication is the SVE governing-predicate qualifier, "M" (merging) or + // "Z" (zeroing). It is set on mask operands of predicated encodings and + // decides whether the generated machine op is the merging or the zeroing + // form (see sveMaskSuffix). + Predication *string + // Governing marks the SVE governing predicate among an instruction's + // operands — the one that selects which lanes the instruction acts on, as + // opposed to a predicate it merely reads as data (SEL's , the / + // of a predicate-logical op). + // + // It is set only where the instruction has no unpredicated encoding, since + // otherwise that encoding carries the operation and its predicated sibling's + // predicate becomes an InVariant instead. So a governing predicate here is + // always one the Go API hides: the generated method, generic op and + // intrinsic omit it, and the lowering synthesizes an all-true predicate in + // its place, which is how predicated-only instructions (ZCMPGT) expose an + // unpredicated API. See #79781. + // + // This is independent of [Operand.Predication]: a governing predicate need + // not carry a qualifier (SADDV
, , . has no lanes to merge + // into), and a qualified predicate need not be governing in this sense (the + // InVariant of a paired operation is a real operand a peephole supplies). + Governing *bool } // VectorSize is a unifier value that is either a number or the string "scalable". @@ -136,10 +162,10 @@ func (vs VectorSize) String() string { return fmt.Sprint(vs.N()) } -// IsImplicitAllTrue reports whether this operand is an SVE governing predicate -// that is dropped from the API and filled with an all-true predicate at lowering. -func (o *Operand) IsImplicitAllTrue() bool { - return o.ImplicitAllTrue != nil && *o.ImplicitAllTrue +// IsGoverning reports whether this operand is the SVE governing predicate, and +// so is dropped from the API and filled with an all-true predicate at lowering. +func (o *Operand) IsGoverning() bool { + return o.Governing != nil && *o.Governing } func (o Operand) OpName(s string) string { diff --git a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go index 79cbe3da76a4e7..bcb2facd2c7b55 100644 --- a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go @@ -194,3 +194,108 @@ func TestStringSVE(t *testing.T) { t.Logf("mx=%s", mx) t.Logf("my=%s", my) } + +//go:noinline +func keepAliveInt8s(archsimd.Int8s) {} + +// TestIfElseSVE checks IfElse and Masked, and that the merging peephole keeps +// the same semantics whether or not it fires: x.Add(y).IfElse(m, x) folds into a +// predicated add, x.Add(y).IfElse(m, z) does not, and both must agree with a +// lane-by-lane reference. +func TestIfElseSVE(t *testing.T) { + if !archsimd.ARM64.SVE() { + t.Skip("no sve") + } + n := archsimd.Int8s{}.Len() + xs, ys, zs := make([]int8, n), make([]int8, n), make([]int8, n) + for i := range xs { + xs[i] = int8(i + 1) + ys[i] = int8(i % 3) // active where xs[i] > ys[i], which alternates early on + zs[i] = int8(-i - 1) + } + x, y, z := archsimd.LoadInt8s(xs), archsimd.LoadInt8s(ys), archsimd.LoadInt8s(zs) + m := x.Greater(y) + + got := make([]int8, n) + check := func(name string, v archsimd.Int8s, want func(i int, active bool) int8) { + t.Helper() + v.Store(got) + for i := 0; i < n; i++ { + if w := want(i, xs[i] > ys[i]); got[i] != w { + t.Errorf("%s: lane %d = %d, want %d (x=%d y=%d)", name, i, got[i], w, xs[i], ys[i]) + } + } + } + + check("IfElse", x.IfElse(m, y), func(i int, active bool) int8 { + if active { + return xs[i] + } + return ys[i] + }) + check("Masked", x.Masked(m), func(i int, active bool) int8 { + if active { + return xs[i] + } + return 0 + }) + // Folds into the merging-predicated add. + check("Add.IfElse(x)", x.Add(y).IfElse(m, x), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return xs[i] + }) + // Folds via commutativity. + check("Add.IfElse(y)", x.Add(y).IfElse(m, y), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return ys[i] + }) + // Folds behind a merging MOVPRFX: the else operand is neither source. + check("Add.IfElse(z)", x.Add(y).IfElse(m, z), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return zs[i] + }) + // ADD has no zeroing-predicated form, so Masked folds into the merging one + // with the zero vector as its else operand. + check("Add.Masked", x.Add(y).Masked(m), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return 0 + }) + + // The prefixed path with every operand still live afterwards, so the + // destination can be none of them and the merging MOVPRFX has to place the + // else operand itself. + rz := x.Add(y).IfElse(m, z) + keepAliveInt8s(x) + keepAliveInt8s(y) + keepAliveInt8s(z) + check("Add.IfElse(z) with all live", rz, func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return zs[i] + }) + + // The MOVPRFX path: x must survive the destructive predicated add. + r := x.Add(y).IfElse(m, x) + keepAliveInt8s(x) + check("Add.IfElse(x) with x live", r, func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return xs[i] + }) + x.Store(got) + for i := 0; i < n; i++ { + if got[i] != xs[i] { + t.Errorf("x clobbered by MOVPRFX: lane %d = %d, want %d", i, got[i], xs[i]) + } + } +} diff --git a/src/simd/archsimd/types_sve.go b/src/simd/archsimd/types_sve.go index 6647aab9037e7d..2c817b381161fe 100644 --- a/src/simd/archsimd/types_sve.go +++ b/src/simd/archsimd/types_sve.go @@ -82,6 +82,21 @@ func (x Float32s) StorePart(s []float32) int { //go:noescape func (x Float32s) storePart(s []float32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Float32s) IfElse(mask Mask32s, y Float32s) Float32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Float32s) Masked(mask Mask32s) Float32s { + var zero Float32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Float32s) String() string { @@ -159,6 +174,21 @@ func (x Float64s) StorePart(s []float64) int { //go:noescape func (x Float64s) storePart(s []float64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Float64s) IfElse(mask Mask64s, y Float64s) Float64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Float64s) Masked(mask Mask64s) Float64s { + var zero Float64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Float64s) String() string { @@ -236,6 +266,21 @@ func (x Int8s) StorePart(s []int8) int { //go:noescape func (x Int8s) storePart(s []int8) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int8s) IfElse(mask Mask8s, y Int8s) Int8s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int8s) Masked(mask Mask8s) Int8s { + var zero Int8s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int8s) String() string { @@ -313,6 +358,21 @@ func (x Int16s) StorePart(s []int16) int { //go:noescape func (x Int16s) storePart(s []int16) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int16s) IfElse(mask Mask16s, y Int16s) Int16s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int16s) Masked(mask Mask16s) Int16s { + var zero Int16s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int16s) String() string { @@ -390,6 +450,21 @@ func (x Int32s) StorePart(s []int32) int { //go:noescape func (x Int32s) storePart(s []int32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int32s) IfElse(mask Mask32s, y Int32s) Int32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int32s) Masked(mask Mask32s) Int32s { + var zero Int32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int32s) String() string { @@ -467,6 +542,21 @@ func (x Int64s) StorePart(s []int64) int { //go:noescape func (x Int64s) storePart(s []int64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int64s) IfElse(mask Mask64s, y Int64s) Int64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int64s) Masked(mask Mask64s) Int64s { + var zero Int64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int64s) String() string { @@ -544,6 +634,21 @@ func (x Uint8s) StorePart(s []uint8) int { //go:noescape func (x Uint8s) storePart(s []uint8) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint8s) IfElse(mask Mask8s, y Uint8s) Uint8s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint8s) Masked(mask Mask8s) Uint8s { + var zero Uint8s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint8s) String() string { @@ -621,6 +726,21 @@ func (x Uint16s) StorePart(s []uint16) int { //go:noescape func (x Uint16s) storePart(s []uint16) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint16s) IfElse(mask Mask16s, y Uint16s) Uint16s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint16s) Masked(mask Mask16s) Uint16s { + var zero Uint16s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint16s) String() string { @@ -698,6 +818,21 @@ func (x Uint32s) StorePart(s []uint32) int { //go:noescape func (x Uint32s) storePart(s []uint32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint32s) IfElse(mask Mask32s, y Uint32s) Uint32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint32s) Masked(mask Mask32s) Uint32s { + var zero Uint32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint32s) String() string { @@ -775,6 +910,21 @@ func (x Uint64s) StorePart(s []uint64) int { //go:noescape func (x Uint64s) storePart(s []uint64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint64s) IfElse(mask Mask64s, y Uint64s) Uint64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint64s) Masked(mask Mask64s) Uint64s { + var zero Uint64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint64s) String() string { diff --git a/test/codegen/simd_arm64.go b/test/codegen/simd_arm64.go index 5911a2d8b654f6..9fc1a2444e7ee8 100644 --- a/test/codegen/simd_arm64.go +++ b/test/codegen/simd_arm64.go @@ -155,9 +155,59 @@ func loToHiUint16Vec(x, lo archsimd.Uint16x8) archsimd.Uint16x8 { return x.ReshapeToUint64s().BitsToFloat64().SetElem(1, lo.ReshapeToUint64s().BitsToFloat64().GetElem(0)).ToBits().ReshapeToUint16s() } +// --- SVE predication peepholes --- +// +// IfElse over an unpredicated operation folds into that operation's +// merging-predicated form. Merging keeps the destination, and an SVE predicated +// instruction is destructive, so an "else" operand that is already one of the +// sources folds into a bare predicated instruction; any other one first needs a +// merging MOVPRFX to put it in the destination. + +func sveIfElseFoldsFirstOperand(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // arm64:`ZADD.*P[0-9]+\.M` -`ZSEL` -`ZMOVPRFX` + return x.Add(y).IfElse(m, x) +} + +func sveIfElseFoldsSecondOperand(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // Commutative, so the mirrored form folds too. + // arm64:`ZADD.*P[0-9]+\.M` -`ZSEL` -`ZMOVPRFX` + return x.Add(y).IfElse(m, y) +} + +func sveIfElseArbitraryElse(x, y, z archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // The else operand is neither source, so a merging MOVPRFX puts it in the + // destination and the destructive add merges over it. + // arm64:`ZMOVPRFX.*P[0-9]+\.M` `ZADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).IfElse(m, z) +} + +func sveMaskedFoldsIntoMerging(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // Masked is a select against zero. ADD has no zeroing-predicated form, so it + // folds into the merging one with the zero vector as the else operand. + // arm64:`ZMOVPRFX.*P[0-9]+\.M` `ZADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).Masked(m) +} + +//go:noinline +func sinkInt8s(archsimd.Int8s) {} + +func sveIfElseMovprfx(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // The else operand is a source, but x stays live so the destructive add + // cannot write it. The whole register is copied, not just the active lanes, + // so this prefix is the unpredicated MOVPRFX. + // arm64:`ZMOVPRFX` `ZADD.*P[0-9]+\.M` -`ZMOVPRFX.*P[0-9]+` + r := x.Add(y).IfElse(m, x) + sinkInt8s(x) + return r +} + +func sveIfElseFloat(x, y archsimd.Float64s, m archsimd.Mask64s) archsimd.Float64s { + // arm64:`ZFADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).IfElse(m, x) +} + // The zero value of a mask is an all-false predicate. func sveZeroMask() archsimd.Mask8s { // arm64:`PPFALSE` -`ZDUP` var m archsimd.Mask8s - return m -} + return m}