From 4756f33d43d84aa3b38e68cad67d24d0720c5a0d Mon Sep 17 00:00:00 2001 From: Junyang Shao Date: Thu, 20 Aug 2026 00:31:04 -0400 Subject: [PATCH] cmd/compile, simd/archsimd: add SVE IfElse and Masked with predication peepholes Add IfElse and Masked to the scalable SVE vector types. IfElse lowers to ZSEL; Masked is a select against a zero vector. The SVE loader now groups an operation's predicated and unpredicated encodings into one definition: the unpredicated encoding (or, for a predicated-only operation, the merging one) carries the operation, and the sibling encodings' governing predicates become inVariants, implying predicated machine ops reached by peephole rather than by an API of their own. An encoding is classified as predicated from the spec's explanation text; see groupPredicationForms in sve/load.go. A select over an unpredicated operation folds into the operation's merging-predicated form: (ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask) with cheaper MOVPRFX-free forms emitted first for a select whose "else" operand is already one of the sources; see sveMergingPeephole in gen_simdrules.go for why the general form requires commutativity. Add showcases only the commutative rules; the machinery also covers non-commutative and predicated-only operations, which arrive in later CLs along the chain. For #79781. Change-Id: I367da550aabee80119c3baa22b0c63381921df8d Reviewed-on: https://go-review.googlesource.com/c/go/+/818460 Reviewed-by: Junyang Shao LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: David Chase --- src/cmd/compile/internal/arm64/simdssa_sve.go | 44 + src/cmd/compile/internal/arm64/ssa.go | 102 ++ src/cmd/compile/internal/ssa/_gen/ARM64.rules | 13 + src/cmd/compile/internal/ssa/_gen/ARM64Ops.go | 8 +- .../compile/internal/ssa/_gen/genericOps.go | 16 + .../internal/ssa/_gen/simdARM64SVE.rules | 42 + .../internal/ssa/_gen/simdARM64SVEops.go | 30 +- src/cmd/compile/internal/ssa/ssaop/opGen.go | 614 ++++++++++ src/cmd/compile/internal/ssagen/intrinsics.go | 18 + .../ssarewrite/rewritearm64/rewriteARM64.go | 1014 +++++++++++++++++ src/simd/archsimd/_gen/simdgen/arch.go | 22 +- .../_gen/simdgen/gen_simdMachineOps.go | 56 + .../archsimd/_gen/simdgen/gen_simdTypes.go | 27 +- .../archsimd/_gen/simdgen/gen_simdrules.go | 100 +- src/simd/archsimd/_gen/simdgen/gen_simdssa.go | 19 +- src/simd/archsimd/_gen/simdgen/gen_utility.go | 31 +- src/simd/archsimd/_gen/simdgen/godefs.go | 205 +++- src/simd/archsimd/_gen/simdgen/sve/emit.go | 115 +- .../archsimd/_gen/simdgen/sve/instruction.go | 99 +- .../_gen/simdgen/sve/instruction_test.go | 128 ++- src/simd/archsimd/_gen/simdgen/sve/load.go | 189 +++ .../archsimd/_gen/simdgen/sve/operands.go | 102 +- .../archsimd/_gen/simdgen/types/operation.go | 46 +- .../internal/simd_test/simd_arm64_test.go | 105 ++ src/simd/archsimd/types_sve.go | 150 +++ test/codegen/simd_arm64.go | 54 +- 26 files changed, 3270 insertions(+), 79 deletions(-) diff --git a/src/cmd/compile/internal/arm64/simdssa_sve.go b/src/cmd/compile/internal/arm64/simdssa_sve.go index 6061c9c3e6ac41..5f184b09c94965 100644 --- a/src/cmd/compile/internal/arm64/simdssa_sve.go +++ b/src/cmd/compile/internal/arm64/simdssa_sve.go @@ -47,6 +47,50 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { case ssaop.OpARM64ZCMPGTS: p = simdZ2kk(s, v, arm64.ARNG_S) + case ssaop.OpARM64ZADDMergingB, + ssaop.OpARM64ZSQADDMergingB, + ssaop.OpARM64ZUQADDMergingB: + p = simdZ2kvPred(s, v, arm64.ARNG_B) + + case ssaop.OpARM64ZFADDMergingD, + ssaop.OpARM64ZADDMergingD, + ssaop.OpARM64ZSQADDMergingD, + ssaop.OpARM64ZUQADDMergingD: + p = simdZ2kvPred(s, v, arm64.ARNG_D) + + case ssaop.OpARM64ZADDMergingH, + ssaop.OpARM64ZSQADDMergingH, + ssaop.OpARM64ZUQADDMergingH: + p = simdZ2kvPred(s, v, arm64.ARNG_H) + + case ssaop.OpARM64ZFADDMergingS, + ssaop.OpARM64ZADDMergingS, + ssaop.OpARM64ZSQADDMergingS, + ssaop.OpARM64ZUQADDMergingS: + p = simdZ2kvPred(s, v, arm64.ARNG_S) + + case ssaop.OpARM64ZADDMergingPrefixedB, + ssaop.OpARM64ZSQADDMergingPrefixedB, + ssaop.OpARM64ZUQADDMergingPrefixedB: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_B) + + case ssaop.OpARM64ZFADDMergingPrefixedD, + ssaop.OpARM64ZADDMergingPrefixedD, + ssaop.OpARM64ZSQADDMergingPrefixedD, + ssaop.OpARM64ZUQADDMergingPrefixedD: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_D) + + case ssaop.OpARM64ZADDMergingPrefixedH, + ssaop.OpARM64ZSQADDMergingPrefixedH, + ssaop.OpARM64ZUQADDMergingPrefixedH: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_H) + + case ssaop.OpARM64ZFADDMergingPrefixedS, + ssaop.OpARM64ZADDMergingPrefixedS, + ssaop.OpARM64ZSQADDMergingPrefixedS, + ssaop.OpARM64ZUQADDMergingPrefixedS: + p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_S) + default: // Unknown reg shape return false diff --git a/src/cmd/compile/internal/arm64/ssa.go b/src/cmd/compile/internal/arm64/ssa.go index 16fd3c4683ac47..70fc8f60d7d3f2 100644 --- a/src/cmd/compile/internal/arm64/ssa.go +++ b/src/cmd/compile/internal/arm64/ssa.go @@ -829,6 +829,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { p.From.Offset = v.AuxInt p.To.Type = obj.TYPE_REG p.To.Reg = v.Reg() + case ssaop.OpARM64ZSELB: + simdZ2kv(s, v, arm64.ARNG_B) + case ssaop.OpARM64ZSELH: + simdZ2kv(s, v, arm64.ARNG_H) + case ssaop.OpARM64ZSELS: + simdZ2kv(s, v, arm64.ARNG_S) + case ssaop.OpARM64ZSELD: + simdZ2kv(s, v, arm64.ARNG_D) case ssaop.OpARM64PWHILELTB: simdPWHILELT(s, v, arm64.ARNG_B) case ssaop.OpARM64PWHILELTH: @@ -2122,6 +2130,100 @@ func simdZ21(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { return p } +// simdZ2kv emits an SVE instruction that takes a predicate as a plain data +// operand rather than a governing predicate, e.g. ZSEL Z1.B, Z0.B, P0, Z2.B. +// Unlike a predicated instruction it is constructive: the destination is +// independent of the sources, so it needs no MOVPRFX. SSA provides arg0=x (kept +// where the predicate is true), arg1=y (kept where false) and arg2=the predicate. +func simdZ2kv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(v.Args[1].Reg(), arng) // Zm + p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn + p.AddRestSourceReg(v.Args[2].Reg()) // Pv, unqualified + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(v.Reg(), arng) // Zd + return p +} + +// simdZ2kvPred emits an SVE predicated binary operation, e.g. ZADD Z1.B, Z0.B, P0.M, +// Z0.B. These instructions are destructive: the destination is also the first +// source. How the destination is put in place depends on what the register +// allocator chose: +// +// - dst == arg0: already destructive-ready, emit the instruction alone. +// - dst == arg1: only reachable for a commutative operation (a +// non-commutative one is marked resultInArg0, which pins dst to arg0), so +// swap the sources and it becomes the case above. +// - anything else: prefix MOVPRFX, which hints the hardware to fuse the pair +// into one constructive operation and leaves both sources intact. +func simdZ2kvPred(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + x, y := v.Args[0].Reg(), v.Args[1].Reg() + d := v.Reg() + switch d { + case x: + case y: + x, y = y, x + default: + mp := s.Prog(arm64.AZMOVPRFX) + mp.From.Type = obj.TYPE_REG + mp.From.Reg = pzreg(x) + mp.To.Type = obj.TYPE_REG + mp.To.Reg = pzreg(d) + } + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(y, arng) // Zm + p.AddRestSourceReg(zregArng(d, arng)) // Zdn + p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M)) // Pg/M + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(d, arng) // Zdn + return p +} + +// simdZ3kvPredResultInArg0 emits an SVE merging-predicated binary operation +// whose inactive lanes come from a value that is neither of its sources, e.g. +// x.Add(y).IfElse(mask, z). SSA provides arg0=z, arg1=x, arg2=y, arg3=mask, and +// resultInArg0 puts z in the destination register. +// +// ZMOVPRFX Zx, Pg/M, Zd // Zd = x on the active lanes, z on the rest +// ZADD Zy, Zd, Pg/M, Zd // Zd = x+y on the active lanes, z on the rest +// +// The prefix must be the predicated MOVPRFX, not the unpredicated one: the +// whole-register form would leave x rather than z in the inactive lanes. +// +// A prefixed instruction may not name its destination in any operand position +// other than the destructive one, so Zd must differ from the operation's Zm. +// Only a commutative operation gets this lowering (see sveMergingPrefixedOp in +// simdgen), so when the register allocator puts the destination on one source — +// which it can only do when z is that source — the other one becomes Zm and the +// prefix is unnecessary. +func simdZ3kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog { + d := v.Reg() + x, y := v.Args[1].Reg(), v.Args[2].Reg() + pg := v.Args[3].Reg() + zn, zm := x, y + if d == zm { + zn, zm = zm, zn + } + if d != zn { + mp := s.Prog(arm64.AZMOVPRFX) + mp.From.Type = obj.TYPE_REG + mp.From.Reg = zregArng(zn, arng) + mp.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) + mp.To.Type = obj.TYPE_REG + mp.To.Reg = zregArng(d, arng) + } + p := s.Prog(v.Op.Asm()) + p.From.Type = obj.TYPE_REG + p.From.Reg = zregArng(zm, arng) // Zm + p.AddRestSourceReg(zregArng(d, arng)) // Zdn + p.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) // Pg/M + p.To.Type = obj.TYPE_REG + p.To.Reg = zregArng(d, arng) // Zdn + return p +} + // simdPWHILELT emits a PWHILELT that fills a predicate with lanes [lo,hi) set for // the given element arrangement, e.g. PWHILELT R0, R1, P0.B. SSA provides // arg0=lo, arg1=hi. diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64.rules b/src/cmd/compile/internal/ssa/_gen/ARM64.rules index 6bde3d674a439f..be80773590db56 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64.rules +++ b/src/cmd/compile/internal/ssa/_gen/ARM64.rules @@ -1905,5 +1905,18 @@ // These are written by hand until the predicated ops are generated by simdgen // in the mask CL; at that point delete them and generate instead. (Count8s r) => (Select0 (PWHILELTB (MOVDconst [0]) r)) + +// SVE per-element select, backing IfElse and Masked. +(IfElseInt8s x mask y) => (ZSELB x y mask) +(IfElseUint8s x mask y) => (ZSELB x y mask) +(IfElseInt16s x mask y) => (ZSELH x y mask) +(IfElseUint16s x mask y) => (ZSELH x y mask) +(IfElseInt32s x mask y) => (ZSELS x y mask) +(IfElseUint32s x mask y) => (ZSELS x y mask) +(IfElseFloat32s x mask y) => (ZSELS x y mask) +(IfElseInt64s x mask y) => (ZSELD x y mask) +(IfElseUint64s x mask y) => (ZSELD x y mask) +(IfElseFloat64s x mask y) => (ZSELD x y mask) + (LoadMasked8 ptr mask mem) && t.Size() == 32 => (ZLD1BPredload ptr mask mem) (StoreMasked8 {t} ptr mask val mem) && t.Size() == 32 => (ZST1BPredstore ptr val mask mem) diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go index 17c4e1c41b2b0e..e0033283c13b45 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go @@ -204,6 +204,8 @@ func init() { fp2flags = regInfo{inputs: []regMask{fp, fp}} fp1flags = regInfo{inputs: []regMask{fp}} fp2predpred = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{pred}} + fp2predfp = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{fp}} + fp3predfp = regInfo{inputs: []regMask{fp, fp, fp, pred}, outputs: []regMask{fp}} predload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{pred}} predstore = regInfo{inputs: []regMask{gpspsbg, pred}} fpload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{fp}} @@ -839,6 +841,10 @@ func init() { // scalable Z bank reuses the fp register masks. {name: "ZLDRload", argLength: 2, reg: fpload, aux: "SymOff", asm: "ZLDR", typ: "Vec256", faultOnNilArg0: true, symEffect: "Read"}, // load from arg0 + auxInt + aux. arg1=mem. {name: "ZSTRstore", argLength: 3, reg: fpstore, aux: "SymOff", asm: "ZSTR", faultOnNilArg0: true, symEffect: "Write"}, // store arg1 to arg0 + auxInt + aux. arg2=mem. + {name: "ZSELB", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELH", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELS", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. + {name: "ZSELD", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive. {name: "PLDRload", argLength: 2, reg: predload, aux: "SymOff", asm: "PLDR", typ: "Mask", faultOnNilArg0: true, symEffect: "Read"}, // load a predicate from arg0 + auxInt + aux. arg1=mem. {name: "PSTRstore", argLength: 3, reg: predstore, aux: "SymOff", asm: "PSTR", faultOnNilArg0: true, symEffect: "Write"}, // store predicate arg1 to arg0 + auxInt + aux. arg2=mem. // PPFALSEB sets every bit of a predicate false, it's the zero value of a predicate. @@ -896,7 +902,7 @@ func init() { pkg: "cmd/internal/obj/arm64", genfile: "../../arm64/ssa.go", genSIMDfile: "../../arm64/simdssa.go ../../arm64/simdssa_sve.go", - ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred)...), + ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred, fp2predfp, fp2predfp, fp3predfp)...), blocks: blocks, regnames: regNamesARM64, ParamIntRegNames: "R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15", diff --git a/src/cmd/compile/internal/ssa/_gen/genericOps.go b/src/cmd/compile/internal/ssa/_gen/genericOps.go index a11a686fd36481..860187b9ad2a31 100644 --- a/src/cmd/compile/internal/ssa/_gen/genericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/genericOps.go @@ -746,6 +746,22 @@ var genericOps = []opData{ {name: "ScalableVectorLen", argLength: 0}, // SVE runtime vector length in bytes {name: "Count8s", argLength: 1}, // arg0 = active byte count; builds an SVE predicate over that many byte lanes + + // IfElse selects per element between two scalable vectors under a predicate. + // It backs both the IfElse and (against a zero vector) the Masked method, and + // is written by hand rather than derived from the ISA because SEL is + // bit-pattern-agnostic: there is no float-lane encoding of it to unify with. + // arg0 = x, arg1 = predicate, arg2 = y (taken where the predicate is false). + {name: "IfElseInt8s", argLength: 3}, + {name: "IfElseUint8s", argLength: 3}, + {name: "IfElseInt16s", argLength: 3}, + {name: "IfElseUint16s", argLength: 3}, + {name: "IfElseInt32s", argLength: 3}, + {name: "IfElseUint32s", argLength: 3}, + {name: "IfElseFloat32s", argLength: 3}, + {name: "IfElseInt64s", argLength: 3}, + {name: "IfElseUint64s", argLength: 3}, + {name: "IfElseFloat64s", argLength: 3}, } // kind controls successors implicit exit diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules index dc7905a5eeb866..d38949935e387a 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules @@ -22,3 +22,45 @@ (GreaterInt32s x y) => (ZCMPGTS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) (GreaterInt64s x y) => (ZCMPGTD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) (GreaterInt8s x y) => (ZCMPGTB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(ZSELB (ZADDB x y) x mask) => (ZADDMergingB x y mask) +(ZSELB (ZADDB x y) y mask) => (ZADDMergingB y x mask) +(ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask) +(ZSELB (ZSQADDB x y) x mask) => (ZSQADDMergingB x y mask) +(ZSELB (ZSQADDB x y) y mask) => (ZSQADDMergingB y x mask) +(ZSELB (ZSQADDB x y) z mask) => (ZSQADDMergingPrefixedB z x y mask) +(ZSELB (ZUQADDB x y) x mask) => (ZUQADDMergingB x y mask) +(ZSELB (ZUQADDB x y) y mask) => (ZUQADDMergingB y x mask) +(ZSELB (ZUQADDB x y) z mask) => (ZUQADDMergingPrefixedB z x y mask) +(ZSELD (ZADDD x y) x mask) => (ZADDMergingD x y mask) +(ZSELD (ZADDD x y) y mask) => (ZADDMergingD y x mask) +(ZSELD (ZADDD x y) z mask) => (ZADDMergingPrefixedD z x y mask) +(ZSELD (ZFADDD x y) x mask) => (ZFADDMergingD x y mask) +(ZSELD (ZFADDD x y) y mask) => (ZFADDMergingD y x mask) +(ZSELD (ZFADDD x y) z mask) => (ZFADDMergingPrefixedD z x y mask) +(ZSELD (ZSQADDD x y) x mask) => (ZSQADDMergingD x y mask) +(ZSELD (ZSQADDD x y) y mask) => (ZSQADDMergingD y x mask) +(ZSELD (ZSQADDD x y) z mask) => (ZSQADDMergingPrefixedD z x y mask) +(ZSELD (ZUQADDD x y) x mask) => (ZUQADDMergingD x y mask) +(ZSELD (ZUQADDD x y) y mask) => (ZUQADDMergingD y x mask) +(ZSELD (ZUQADDD x y) z mask) => (ZUQADDMergingPrefixedD z x y mask) +(ZSELH (ZADDH x y) x mask) => (ZADDMergingH x y mask) +(ZSELH (ZADDH x y) y mask) => (ZADDMergingH y x mask) +(ZSELH (ZADDH x y) z mask) => (ZADDMergingPrefixedH z x y mask) +(ZSELH (ZSQADDH x y) x mask) => (ZSQADDMergingH x y mask) +(ZSELH (ZSQADDH x y) y mask) => (ZSQADDMergingH y x mask) +(ZSELH (ZSQADDH x y) z mask) => (ZSQADDMergingPrefixedH z x y mask) +(ZSELH (ZUQADDH x y) x mask) => (ZUQADDMergingH x y mask) +(ZSELH (ZUQADDH x y) y mask) => (ZUQADDMergingH y x mask) +(ZSELH (ZUQADDH x y) z mask) => (ZUQADDMergingPrefixedH z x y mask) +(ZSELS (ZADDS x y) x mask) => (ZADDMergingS x y mask) +(ZSELS (ZADDS x y) y mask) => (ZADDMergingS y x mask) +(ZSELS (ZADDS x y) z mask) => (ZADDMergingPrefixedS z x y mask) +(ZSELS (ZFADDS x y) x mask) => (ZFADDMergingS x y mask) +(ZSELS (ZFADDS x y) y mask) => (ZFADDMergingS y x mask) +(ZSELS (ZFADDS x y) z mask) => (ZFADDMergingPrefixedS z x y mask) +(ZSELS (ZSQADDS x y) x mask) => (ZSQADDMergingS x y mask) +(ZSELS (ZSQADDS x y) y mask) => (ZSQADDMergingS y x mask) +(ZSELS (ZSQADDS x y) z mask) => (ZSQADDMergingPrefixedS z x y mask) +(ZSELS (ZUQADDS x y) x mask) => (ZUQADDMergingS x y mask) +(ZSELS (ZUQADDS x y) y mask) => (ZUQADDMergingS y x mask) +(ZSELS (ZUQADDS x y) z mask) => (ZUQADDMergingPrefixedS z x y mask) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go index 98a2164e1fb8d4..fd3453977bda51 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go @@ -2,25 +2,53 @@ package main -func simdARM64SVEOps(z11, z21, z2kk regInfo) []opData { +func simdARM64SVEOps(z11, z21, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []opData { return []opData{ {name: "ZADDB", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDD", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDH", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, + {name: "ZADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZADD", typ: "Vec256", resultInArg0: true}, + {name: "ZADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZADDS", argLength: 2, reg: z21, asm: "ZADD", commutative: true, typ: "Vec256"}, {name: "ZCMPGTB", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTD", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTH", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZCMPGTS", argLength: 3, reg: z2kk, asm: "ZCMPGT", typ: "Mask"}, {name: "ZFADDD", argLength: 2, reg: z21, asm: "ZFADD", commutative: true, typ: "Vec256"}, + {name: "ZFADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZFADD", commutative: true, typ: "Vec256"}, + {name: "ZFADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZFADD", typ: "Vec256", resultInArg0: true}, + {name: "ZFADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZFADD", typ: "Vec256", resultInArg0: true}, + {name: "ZFADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZFADD", commutative: true, typ: "Vec256"}, {name: "ZFADDS", argLength: 2, reg: z21, asm: "ZFADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDB", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDD", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDH", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, + {name: "ZSQADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZSQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZSQADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDS", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDB", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDD", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDH", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingB", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingD", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingH", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, + {name: "ZUQADDMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZUQADD", typ: "Vec256", resultInArg0: true}, + {name: "ZUQADDMergingS", argLength: 3, reg: z2kvPred, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDS", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, } } diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index debf9d284726f2..a1b687e63b8487 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -4715,6 +4715,10 @@ const ( OpARM64VMOVI16B OpARM64ZLDRload OpARM64ZSTRstore + OpARM64ZSELB + OpARM64ZSELH + OpARM64ZSELS + OpARM64ZSELD OpARM64PLDRload OpARM64PSTRstore OpARM64PPFALSEB @@ -5039,20 +5043,48 @@ const ( OpARM64ZADDB OpARM64ZADDD OpARM64ZADDH + OpARM64ZADDMergingB + OpARM64ZADDMergingD + OpARM64ZADDMergingH + OpARM64ZADDMergingPrefixedB + OpARM64ZADDMergingPrefixedD + OpARM64ZADDMergingPrefixedH + OpARM64ZADDMergingPrefixedS + OpARM64ZADDMergingS OpARM64ZADDS OpARM64ZCMPGTB OpARM64ZCMPGTD OpARM64ZCMPGTH OpARM64ZCMPGTS OpARM64ZFADDD + OpARM64ZFADDMergingD + OpARM64ZFADDMergingPrefixedD + OpARM64ZFADDMergingPrefixedS + OpARM64ZFADDMergingS OpARM64ZFADDS OpARM64ZSQADDB OpARM64ZSQADDD OpARM64ZSQADDH + OpARM64ZSQADDMergingB + OpARM64ZSQADDMergingD + OpARM64ZSQADDMergingH + OpARM64ZSQADDMergingPrefixedB + OpARM64ZSQADDMergingPrefixedD + OpARM64ZSQADDMergingPrefixedH + OpARM64ZSQADDMergingPrefixedS + OpARM64ZSQADDMergingS OpARM64ZSQADDS OpARM64ZUQADDB OpARM64ZUQADDD OpARM64ZUQADDH + OpARM64ZUQADDMergingB + OpARM64ZUQADDMergingD + OpARM64ZUQADDMergingH + OpARM64ZUQADDMergingPrefixedB + OpARM64ZUQADDMergingPrefixedD + OpARM64ZUQADDMergingPrefixedH + OpARM64ZUQADDMergingPrefixedS + OpARM64ZUQADDMergingS OpARM64ZUQADDS OpLOONG64NEGV @@ -6962,6 +6994,16 @@ const ( OpIsNaNFloat64x8 OpScalableVectorLen OpCount8s + OpIfElseInt8s + OpIfElseUint8s + OpIfElseInt16s + OpIfElseUint16s + OpIfElseInt32s + OpIfElseUint32s + OpIfElseFloat32s + OpIfElseInt64s + OpIfElseUint64s + OpIfElseFloat64s OpAESDecryptLastRoundUint8x16 OpAESDecryptLastRoundUint8x32 OpAESDecryptLastRoundUint8x64 @@ -81052,6 +81094,66 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZSELB", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELH", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELS", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSELD", + ArgLen: 3, + asm: arm64.AZSEL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "PLDRload", AuxType: AuxTypeSymOff, @@ -85621,6 +85723,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZADDS", ArgLen: 2, @@ -85711,6 +85945,72 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZFADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZFADDS", ArgLen: 2, @@ -85771,6 +86071,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZSQADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSQADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZSQADDS", ArgLen: 2, @@ -85831,6 +86263,138 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZUQADDMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUQADDMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUQADD, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZUQADDS", ArgLen: 2, @@ -109275,6 +109839,56 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 1, Generic: true, }, + { + Name: "IfElseInt8s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint8s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt16s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint16s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseFloat32s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseInt64s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseUint64s", + ArgLen: 3, + Generic: true, + }, + { + Name: "IfElseFloat64s", + ArgLen: 3, + Generic: true, + }, { Name: "AESDecryptLastRoundUint8x16", ArgLen: 2, diff --git a/src/cmd/compile/internal/ssagen/intrinsics.go b/src/cmd/compile/internal/ssagen/intrinsics.go index 0d93a9ee5c63bf..6a0c4e41a32368 100644 --- a/src/cmd/compile/internal/ssagen/intrinsics.go +++ b/src/cmd/compile/internal/ssagen/intrinsics.go @@ -1700,6 +1700,24 @@ func initIntrinsics(cfg *intrinsicBuildConfig) { addF(simdPackage, "load"+t.name+"Part", sveLoadPart(t.bytes), sys.ARM64) addF(simdPackage, t.name+".storePart", sveStorePart(t.bytes), sys.ARM64) } + // IfElse backs both the IfElse and Masked methods on every scalable vector. + for _, t := range []struct { + name string + op ssaop.Op + }{ + {"Int8s", ssaop.OpIfElseInt8s}, + {"Uint8s", ssaop.OpIfElseUint8s}, + {"Int16s", ssaop.OpIfElseInt16s}, + {"Uint16s", ssaop.OpIfElseUint16s}, + {"Int32s", ssaop.OpIfElseInt32s}, + {"Uint32s", ssaop.OpIfElseUint32s}, + {"Float32s", ssaop.OpIfElseFloat32s}, + {"Int64s", ssaop.OpIfElseInt64s}, + {"Uint64s", ssaop.OpIfElseUint64s}, + {"Float64s", ssaop.OpIfElseFloat64s}, + } { + addF(simdPackage, t.name+".IfElse", opLen3(t.op, types.TypeVec256), sys.ARM64) + } addF(simdPackage, "ClearAVXUpperBits", func(s *state, n *ir.CallExpr, args []*ssa.Value) *ssa.Value { diff --git a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go index be73f1e02f3039..b9f150810ce6d7 100644 --- a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go +++ b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go @@ -519,6 +519,14 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpARM64XORshiftRL(v) case ssaop.OpARM64XORshiftRO: return rewriteValue_OpARM64XORshiftRO(v) + case ssaop.OpARM64ZSELB: + return rewriteValue_OpARM64ZSELB(v) + case ssaop.OpARM64ZSELD: + return rewriteValue_OpARM64ZSELD(v) + case ssaop.OpARM64ZSELH: + return rewriteValue_OpARM64ZSELH(v) + case ssaop.OpARM64ZSELS: + return rewriteValue_OpARM64ZSELS(v) case ssaop.OpAbs: v.Op = ssaop.OpARM64FABSD return true @@ -1331,6 +1339,26 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpHmul64u: v.Op = ssaop.OpARM64UMULH return true + case ssaop.OpIfElseFloat32s: + return rewriteValue_OpIfElseFloat32s(v) + case ssaop.OpIfElseFloat64s: + return rewriteValue_OpIfElseFloat64s(v) + case ssaop.OpIfElseInt16s: + return rewriteValue_OpIfElseInt16s(v) + case ssaop.OpIfElseInt32s: + return rewriteValue_OpIfElseInt32s(v) + case ssaop.OpIfElseInt64s: + return rewriteValue_OpIfElseInt64s(v) + case ssaop.OpIfElseInt8s: + return rewriteValue_OpIfElseInt8s(v) + case ssaop.OpIfElseUint16s: + return rewriteValue_OpIfElseUint16s(v) + case ssaop.OpIfElseUint32s: + return rewriteValue_OpIfElseUint32s(v) + case ssaop.OpIfElseUint64s: + return rewriteValue_OpIfElseUint64s(v) + case ssaop.OpIfElseUint8s: + return rewriteValue_OpIfElseUint8s(v) case ssaop.OpInterCall: v.Op = ssaop.OpARM64CALLinter return true @@ -19812,6 +19840,842 @@ func rewriteValue_OpARM64XORshiftRO(v *ssa.Value) bool { } return false } +func rewriteValue_OpARM64ZSELB(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELB (ZADDB x y) x mask) + // result: (ZADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZADDB x y) y mask) + // result: (ZADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZADDB x y) z mask) + // result: (ZADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELB (ZSQADDB x y) x mask) + // result: (ZSQADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZSQADDB x y) y mask) + // result: (ZSQADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZSQADDB x y) z mask) + // result: (ZSQADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELB (ZUQADDB x y) x mask) + // result: (ZUQADDMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZUQADDB x y) y mask) + // result: (ZUQADDMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZUQADDB x y) z mask) + // result: (ZUQADDMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDB { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELD (ZADDD x y) x mask) + // result: (ZADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZADDD x y) y mask) + // result: (ZADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZADDD x y) z mask) + // result: (ZADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZFADDD x y) x mask) + // result: (ZFADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZFADDD x y) y mask) + // result: (ZFADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZFADDD x y) z mask) + // result: (ZFADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZSQADDD x y) x mask) + // result: (ZSQADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) y mask) + // result: (ZSQADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) z mask) + // result: (ZSQADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZUQADDD x y) x mask) + // result: (ZUQADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZUQADDD x y) y mask) + // result: (ZUQADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZUQADDD x y) z mask) + // result: (ZUQADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELH(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELH (ZADDH x y) x mask) + // result: (ZADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZADDH x y) y mask) + // result: (ZADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZADDH x y) z mask) + // result: (ZADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELH (ZSQADDH x y) x mask) + // result: (ZSQADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZSQADDH x y) y mask) + // result: (ZSQADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZSQADDH x y) z mask) + // result: (ZSQADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELH (ZUQADDH x y) x mask) + // result: (ZUQADDMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZUQADDH x y) y mask) + // result: (ZUQADDMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZUQADDH x y) z mask) + // result: (ZUQADDMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDH { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } + return false +} +func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (ZSELS (ZADDS x y) x mask) + // result: (ZADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZADDS x y) y mask) + // result: (ZADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZADDS x y) z mask) + // result: (ZADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZFADDS x y) x mask) + // result: (ZFADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZFADDS x y) y mask) + // result: (ZFADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZFADDS x y) z mask) + // result: (ZFADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZSQADDS x y) x mask) + // result: (ZSQADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZSQADDS x y) y mask) + // result: (ZSQADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZSQADDS x y) z mask) + // result: (ZSQADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZUQADDS x y) x mask) + // result: (ZUQADDMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZUQADDS x y) y mask) + // result: (ZUQADDMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZUQADDS x y) z mask) + // result: (ZUQADDMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUQADDS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUQADDMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + return false +} func rewriteValue_OpAddr(v *ssa.Value) bool { v_0 := v.Args[0] // match: (Addr {sym} base) @@ -20546,6 +21410,156 @@ func rewriteValue_OpHmul32u(v *ssa.Value) bool { return true } } +func rewriteValue_OpIfElseFloat32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseFloat32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseFloat64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseFloat64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt16s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt16s x mask y) + // result: (ZSELH x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELH) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseInt8s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseInt8s x mask y) + // result: (ZSELB x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELB) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint16s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint16s x mask y) + // result: (ZSELH x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELH) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint32s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint32s x mask y) + // result: (ZSELS x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELS) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint64s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint64s x mask y) + // result: (ZSELD x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELD) + v.AddArg3(x, y, mask) + return true + } +} +func rewriteValue_OpIfElseUint8s(v *ssa.Value) bool { + v_2 := v.Args[2] + v_1 := v.Args[1] + v_0 := v.Args[0] + // match: (IfElseUint8s x mask y) + // result: (ZSELB x y mask) + for { + x := v_0 + mask := v_1 + y := v_2 + v.Reset(ssaop.OpARM64ZSELB) + v.AddArg3(x, y, mask) + return true + } +} func rewriteValue_OpIsInBounds(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/simd/archsimd/_gen/simdgen/arch.go b/src/simd/archsimd/_gen/simdgen/arch.go index 608805f4c6d446..cd6d7284620252 100644 --- a/src/simd/archsimd/_gen/simdgen/arch.go +++ b/src/simd/archsimd/_gen/simdgen/arch.go @@ -164,18 +164,26 @@ var sveArrangements = []string{"B", "H", "S", "D"} // predicated ops are supported. The names are the parameters of the generated // simdARM64SVEOps function, bound to concrete regInfo values in ARM64Ops.go. var sveRegInfoKeys = []string{ - "z11", // 1 Z in, 1 Z out (unary, e.g. NEG) - "z21", // 2 Z in, 1 Z out (binary, e.g. unpredicated ADD) - "z2kk", // 2 Z in, 1 P (governing predicate) in, 1 P out (predicated compare, e.g. ZCMPGT) + "z11", // 1 Z in, 1 Z out (unary, e.g. NEG) + "z21", // 2 Z in, 1 Z out (binary, e.g. unpredicated ADD) + "z2kk", // 2 Z in, 1 P (governing predicate) in, 1 P out (predicated compare, e.g. ZCMPGT) + "z2kv", // 2 Z in, 1 P (select predicate) in, 1 Z out (constructive, e.g. ZSEL) + "z2kvPred", // 2 Z in, 1 P (governing predicate) in, 1 Z out (destructive, e.g. ZADD/M) + // 3 Z in, 1 P (governing predicate) in, 1 Z out, destination shared with the + // first input: a destructive predicated op behind a MOVPRFX, e.g. ZADDMergingPrefixed. + "z3kvPredResultInArg0", } var sveRegInfoSet = map[string]bool{ - "z11": true, - "z21": true, - "z2kk": true, + "z11": true, + "z21": true, + "z2kk": true, + "z2kv": true, + "z2kvPred": true, + "z3kvPred": true, } -const sveRegInfoParams = "z11, z21, z2kk regInfo" +const sveRegInfoParams = "z11, z21, z2kk, z2kv, z2kvPred, z3kvPred regInfo" const sveGeneratedHeader = `// Code generated by 'simdgen -o godefs -goroot $GOROOT -arch sve -arm64Path $ARM64_ISA_PATH go_sve.yaml types.yaml categories.yaml'; DO NOT EDIT. ` diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go index 246d969cd4bd94..65b85612ffb1b7 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdMachineOps.go @@ -163,6 +163,24 @@ func writeSIMDMachineOps(buffer *bytes.Buffer, ops []Operation) { if shapeOut == OneVregOutAtIn { resultInArg0 = true } + if CurrentArch().isSVE() { + switch idx := gOp.sveInPlaceInput(); { + case idx < 0: + // Constructive: the destination is independent of the sources. + case idx == 0: + // The instruction overwrites its first source. A commutative one is + // left unconstrained — the ssa-to-prog helper puts the destination in + // place by swapping the operands or by prefixing a MOVPRFX. A + // non-commutative one cannot be fixed by swapping, so pin the + // destination to the first source instead. + if !gOp.Commutative { + resultInArg0 = true + } + default: + panic(fmt.Errorf("simdgen: %s overwrites input %d; only the first input is supported: %s", + gOp.Asm, idx, gOp)) + } + } var memOpData *opData regInfoMerging := regInfo hasMerging := false @@ -214,6 +232,44 @@ func writeSIMDMachineOps(buffer *bytes.Buffer, ops []Operation) { } } else { opsData = append(opsData, opData{asm, gOp.Asm, len(gOp.In), regInfo, gOp.Commutative, outType, resultInArg0}) + // The inVariant implies machine ops only: one predicated instruction + // per governing-predicate qualifier the encoding supports, reached by + // peephole rather than by any API of its own. + for _, pred := range gOp.svePredicatedOps() { + predRegInfo, err := makeRegInfo(pred, NoMem) + if err != nil { + panic(err) + } + predResultInArg0 := false + switch idx := pred.sveInPlaceInput(); { + case idx < 0: + case idx == 0: + // Where the first input is the merge source it is the whole + // reason the destination is pinned, so commutativity — which + // is about the two sources — does not enter into it. + predResultInArg0 = pred.sveMergeSourceIn0 || !pred.Commutative + default: + panic(fmt.Errorf("simdgen: %s overwrites input %d; only the first input is supported: %s", + pred.Asm, idx, pred)) + } + opsData = append(opsData, opData{machineOpName(OneMask, pred), pred.Asm, len(pred.In), + predRegInfo, pred.Commutative, outType, predResultInArg0}) + // There is no zeroing machine op here for Masked to fold into: + // every ARM64 instruction that has both an unpredicated and a + // predicated encoding is /M-only. The /Z forms belong to + // predicated-only instructions (ABS, NEG, NOT, ...), where + // sveImplicitPredPeepholes folds Masked into them. + if prefixed := pred.sveMergingPrefixedOp(); prefixed != nil { + prefixedRegInfo, err := makeRegInfo(*prefixed, NoMem) + if err != nil { + panic(err) + } + // The extra input is the value the destination starts out + // holding, so the destination must share its register. + opsData = append(opsData, opData{machineOpName(OneMask, *prefixed), prefixed.Asm, len(prefixed.In), + prefixedRegInfo, false, outType, true}) + } + } if memOpData != nil { if *op.MemFeatures != "vbcst" { panic("simdgen only knows vbcst for mem ops for now") diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go index 9353fbb6dfcc56..6f72183dfeff62 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdTypes.go @@ -369,6 +369,23 @@ func (m {{.Name}}) Store(bits []uint16) { func (m {{.Name}}) store(bits []uint16) {{end}} +{{define "sveIfElseTmpl"}} +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x {{.Name}}) IfElse(mask Mask{{.ElemBits}}s, y {{.Name}}) {{.Name}} + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x {{.Name}}) Masked(mask Mask{{.ElemBits}}s) {{.Name}} { + var zero {{.Name}} + return x.IfElse(mask, zero) +} +{{end}} + {{define "sveStringTmpl"}} {{- if eq .Type "mask"}} // String returns a string representation of SIMD mask m: 1 for an active lane, @@ -944,9 +961,13 @@ type psve struct { } } } - // Scalable types print only the lanes that exist at the runtime vector - // length, so their String is generated here rather than by tmplgen (which - // generates the fixed-width ones from a constant lane count). + // TODO: these type utility methods can also be generated by tmplgen, or we can move other arches from + // tmplgen to here. + if typeDef.IsScalable() && typeDef.Type() != "mask" { + if err := t.ExecuteTemplate(buffer, "sveIfElseTmpl", typeDef); err != nil { + panic(fmt.Errorf("failed to execute sveIfElseTmpl template for type %s: %w", typeDef.Name(), err)) + } + } if typeDef.IsScalable() { if err := t.ExecuteTemplate(buffer, "sveStringTmpl", typeDef); err != nil { panic(fmt.Errorf("failed to execute sveStringTmpl template for type %s: %w", typeDef.Name(), err)) diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go index a1c0ecb552f9e2..1eebd1d2afd7dc 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go @@ -198,7 +198,7 @@ func expandFormatSpecifiers(s string, elemBits int) string { // the predicated machine op with a synthesized all-true predicate. // // The all-true predicate is PWHILELT(0, lanes) with lanes = -// maxVectorBits/elemBits, the lane count at the maximum supported vector length. +// MaxVectorBits/elemBits, the lane count at the maximum supported vector length. // Since PWHILELT saturates // (lane i is set while i < hi), this predicate is all-true at any smaller VL too, // so it stands in for the not-yet-available PTRUE. @@ -214,6 +214,93 @@ func sveImplicitPredRule(gOp Operation, asm, args string) string { gOp.GenericName(), args, asm, args, letter, lanes) } +// sveAllTruePattern returns the rule text for the synthesized all-true +// governing predicate of an operation, the same term sveImplicitPredRule +// produces. Matching it, rather than a wildcard, is what makes the peepholes +// below sound: they replace the predicate, so they may only fire on one that +// selects every lane. +func sveAllTruePattern(gOp Operation) string { + elemBits := *gOp.Out[0].ElemBits + return fmt.Sprintf("(Select0 (PWHILELT%s (MOVDconst [0]) (MOVDconst [%d])))", + sveArrangementLetter(gOp), types.MaxVectorBits/elemBits) +} + +// sveImplicitPredPeepholes returns the rules that fold a select over an +// operation whose governing predicate is implicit-all-true into that +// operation's genuinely predicated forms. The operation computes every lane and +// the select then throws most of them away, so the predicate the select +// describes can simply take the place of the all-true one: +// +// (ZSELB (ZABSB x ) z mask) => (ZABSMergingB z x mask) +// +// The select's "else" operand stays in the inactive lanes, which is what merging +// predication does; naming that operand is something a constructive instruction +// does natively, so no MOVPRFX is involved. Masked -- a select against zero -- +// folds through the same rule, with the zero vector as the else operand. +func sveImplicitPredPeepholes(gOp Operation, asm, args string) string { + if governingInput(gOp.In) < 0 { + return "" + } + sel := "ZSEL" + sveArrangementLetter(gOp) + var rules string + for _, pred := range gOp.svePredicatedOps() { + if sveMaskSuffix(pred) != "Merging" || !pred.sveMergeSourceIn0 { + continue + } + rules += fmt.Sprintf("(%s (%s %s %s) z mask) => (%s z %s mask)\n", + sel, asm, args, sveAllTruePattern(gOp), machineOpName(OneMask, pred), args) + } + return rules +} + +// sveMergingPeephole returns the rules that fold a select over an unpredicated +// SVE operation into the operation's merging-predicated form, e.g. +// +// (ZSELB (ZADDB x y) x mask) => (ZADDMergingB x y mask) +// +// which is what x.Add(y).IfElse(mask, x) lowers to. Merging predication keeps +// the destination — and an SVE predicated instruction is destructive, so the +// destination is the first source — which is why the select's "else" operand +// must be that same operand for this shortest form to hold. A commutative +// operation gets the mirrored rule too, since either source can play that role. +// +// A commutative operation also gets the general rule, whose "else" operand is +// unrestricted because the lowering prefixes a MOVPRFX: +// +// (ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask) +// +// It is emitted last so the two rules above win where they apply and save the +// prefix. See [Operation.sveMergingPrefixedOp] for why it needs commutativity. +func sveMergingPeephole(gOp Operation) string { + var pred *types.Operand + vregs := 0 + for i := range gOp.In { + switch { + case gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && !gOp.In[i].IsGoverning(): + pred = &gOp.In[i] + case gOp.In[i].Class == "vreg": + vregs++ + } + } + if pred == nil || *pred.Predication != "M" || vregs != 2 { + return "" + } + sel := "ZSEL" + sveArrangementLetter(gOp) + unpred := machineOpName(NoMask, gOp) + merging := machineOpName(OneMask, gOp) + rules := fmt.Sprintf("(%s (%s x y) x mask) => (%s x y mask)\n", sel, unpred, merging) + if gOp.Commutative { + rules += fmt.Sprintf("(%s (%s x y) y mask) => (%s y x mask)\n", sel, unpred, merging) + } + if gOp.sveMergingPrefixedOp() != nil { + // The general case: any "else" operand, reached by prefixing a MOVPRFX. + // It subsumes the two rules above, which come first so that a select + // whose "else" is already one of the sources keeps the shorter encoding. + rules += fmt.Sprintf("(%s (%s x y) z mask) => (%s z x y mask)\n", sel, unpred, machineOpName(OneMask, *gOp.sveMergingPrefixedOp())) + } + return rules +} + // writeSIMDRules generates the lowering and rewrite rules for ssa and writes it to simdAMD64.rules // within the specified directory. func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { @@ -275,6 +362,9 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { // mask-conversion machinery below (SVE predicates are represented as-is). if opr.implicitPredCount() > 0 { sveRules = append(sveRules, sveImplicitPredRule(gOp, asm, data.Args)) + if r := sveImplicitPredPeepholes(gOp, asm, data.Args); r != "" { + sveRules = append(sveRules, r) + } asmCheck[asm] = true continue } @@ -498,6 +588,11 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { data.ArgsOut = "..." } data.TplName = tplName + for _, pred := range gOp.svePredicatedOps() { + if r := sveMergingPeephole(pred); r != "" { + sveRules = append(sveRules, r) + } + } if opr.NoGenericOps != nil && *opr.NoGenericOps == "true" || opr.SkipMaskedMethod() { optData = append(optData, data) @@ -524,7 +619,10 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { } } + // Signed and unsigned element types share machine ops, so the same rule can + // be produced more than once. slices.Sort(sveRules) + sveRules = slices.Compact(sveRules) for _, rule := range sveRules { buffer.WriteString(rule) } diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go index 6dbf5a43094537..c128bc0cc2b422 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdssa.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdssa.go @@ -191,6 +191,19 @@ func writeSIMDSSA(buffer *bytes.Buffer, ops []Operation) { registerRegShape(regShape, caseStr, op) return nil } + // An SVE inVariant implies machine ops that are not operations of their own, + // so expand them here too; each needs its own ssa-to-prog case. + expanded := make([]Operation, 0, len(ops)) + for _, op := range ops { + expanded = append(expanded, op) + for _, pred := range op.svePredicatedOps() { + expanded = append(expanded, pred) + if prefixed := pred.sveMergingPrefixedOp(); prefixed != nil { + expanded = append(expanded, *prefixed) + } + } + } + ops = expanded for _, op := range ops { shapeIn, shapeOut, maskType, immType, gOp, immOpArg := op.shape() asm := machineOpName(maskType, gOp) @@ -201,7 +214,11 @@ func writeSIMDSSA(buffer *bytes.Buffer, ops []Operation) { caseStr := fmt.Sprintf("ssaop.Op%s%s", archInfo.ArchUpper, asm) isZeroMasking := false if shapeIn == OneKmaskIn || shapeIn == OneKmaskImmIn { - if gOp.Zeroing == nil || *gOp.Zeroing { + if (gOp.Zeroing == nil || *gOp.Zeroing) && !CurrentArch().isSVE() { + // x86 spells the zeroing/merging choice as an assembler suffix on + // the masked instruction. SVE encodes it in the governing predicate + // operand itself (Pg/Z or Pg/M), which the ssa-to-prog helper + // already emits, so there is no suffix to parse. ZeroingMask = append(ZeroingMask, caseStr) isZeroMasking = true } diff --git a/src/simd/archsimd/_gen/simdgen/gen_utility.go b/src/simd/archsimd/_gen/simdgen/gen_utility.go index bd78c71b6ee7fb..44f2ba629cc1b0 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_utility.go +++ b/src/simd/archsimd/_gen/simdgen/gen_utility.go @@ -165,7 +165,7 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS hasVreg := false hasListIn := false for _, in := range op.In { - if in.IsImplicitAllTrue() { + if in.IsGoverning() { // An SVE implicit-all-true governing predicate is not part of the Go // API: it must not count as a mask input here, so the op classifies as // an unpredicated (PureVregIn/NoMask) op. The machine op and lowering @@ -194,8 +194,14 @@ func (op *Operation) shape() (shapeIn inShape, shapeOut outShape, maskType maskS if immAsmPos == outputReg { immOpIdx += "Out" } - } else if in.Class == "mask" { + } else if in.Class == "mask" && (!CurrentArch().isSVE() || in.Predication != nil) { maskCount++ + } else if in.Class == "mask" { + // An SVE predicate operand with no /M or /Z qualifier is , a plain + // data operand (SEL's select predicate) rather than a governing + // predicate: the operation is unpredicated and the mask is just an + // argument. regShape still counts it as a predicate register. + hasVreg = true } else { if immAsmPos == in.AsmPos { immOpIdx += fmt.Sprintf("In%d", in.AsmPos) @@ -356,6 +362,20 @@ func (op *Operation) regShape(mem memShape) (string, error) { panic("simdgen does not understand memory as output as of now") } regInfo += fixedName + if CurrentArch().isSVE() { + // A governing predicate supplied by the caller (/M or /Z, from the paired + // predicated encoding) means the instruction is predicated and + // destructive; a plain predicate operand (SEL's ) is not, and an + // implicit-all-true predicate is synthesized rather than passed in. They + // share register classes but need different ssa-to-prog helpers, so give + // the caller-predicated form its own shape name. + for i := range gOp.In { + if gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && !gOp.In[i].IsGoverning() { + regInfo += "Pred" + break + } + } + } if CurrentArch().isSVE() && strings.HasPrefix(regInfo, "v") { // SVE vectors live in the scalable Z bank, not the NEON V bank, so name // their shapes with a "z" (z21, z11, ...). This keeps the generated @@ -941,6 +961,13 @@ func (o *Operation) hasMaskedMerging(maskType maskShape, outType outShape) bool } } } + if CurrentArch().isSVE() { + // AMD64 merging takes the merge source as an extra destination operand. + // An SVE predicated instruction is destructive — it merges into its own + // first source — so there is no separate operand to add, and the merging + // form is just the /M-predicated machine op. + return false + } // BLEND and VMOVDQU are not user-facing ops so we should filter them out. return o.OperandOrder == nil && maskType == OneMask && outType == OneVregOut && len(o.InVariant) == 1 && !strings.Contains(o.Asm, "BLEND") && !strings.Contains(o.Asm, "VMOVDQU") diff --git a/src/simd/archsimd/_gen/simdgen/godefs.go b/src/simd/archsimd/_gen/simdgen/godefs.go index fcdb1ee0c66401..c1c481cc60c60a 100644 --- a/src/simd/archsimd/_gen/simdgen/godefs.go +++ b/src/simd/archsimd/_gen/simdgen/godefs.go @@ -44,6 +44,19 @@ type Operation struct { // // For masked operations, this will have the mask operand appended. In []types.Operand + + // sveMergingPrefixed marks the MOVPRFX-prefixed variant of a merging + // predicated operation, built by [Operation.sveMergingPrefixedOp]. It exists + // only to give that variant a machine-op name of its own. + sveMergingPrefixed bool + + // sveMergeSourceIn0 marks a merging predicated operation whose first input + // is the value the destination starts out holding, and which therefore has + // to share that input's register. Merging predication leaves the inactive + // lanes of the destination alone, so that value is an operand of the + // operation whether the instruction names it (a constructive one does, as + // ABS , /M, ) or a MOVPRFX has to put it there. + sveMergeSourceIn0 bool } func (o *Operation) IsMasked() bool { @@ -106,6 +119,13 @@ func (o *Operation) DecodeUnified(v *unify.Value) error { } isMasked := o.IsMasked() + if CurrentArch().isSVE() { + // An SVE inVariant is the operation's predicated encoding, not a separate + // masked API. The operation keeps its unpredicated name and inputs; the + // predicate is picked up later, by the machine op and peephole generators, + // through svePredicated. + isMasked = false + } // Compute full Go method name. o.Go = o.rawOperation.Go @@ -133,7 +153,10 @@ func (o *Operation) DecodeUnified(v *unify.Value) error { o.Documentation += "\n" + reForName.ReplaceAllString(*o.rawOperation.AddDoc, o.Go) } - o.In = append(o.rawOperation.In, o.rawOperation.InVariant...) + o.In = o.rawOperation.In + if !CurrentArch().isSVE() { + o.In = append(o.rawOperation.In, o.rawOperation.InVariant...) + } // For operations that read only the lower half of input registers (indicated by hiHalfAsm), // add a doc note showing the compositional pattern for the upper half. @@ -205,6 +228,19 @@ var demotingConvertOps = map[string]bool{ "VPMOVWBMasked128": true, "VPMOVSWBMasked128": true, "VPMOVUSWBMasked128": true, } +// sveMaskSuffix returns the machine-op name suffix for a masked operation: +// "Merging" for an SVE /M predicate, "Masked" for /Z and for every other target. +func sveMaskSuffix(gOp Operation) string { + if CurrentArch().isSVE() { + for i := range gOp.In { + if gOp.In[i].Class == "mask" && gOp.In[i].Predication != nil && *gOp.In[i].Predication == "M" { + return "Merging" + } + } + } + return "Masked" +} + // sveArrangementLetter returns the SVE element-size arrangement letter // (B=8, H=16, S=32, D=64) that names an SVE machine op, or "" when the target // is not SVE. The letter comes from the operation's governing element width: @@ -243,7 +279,15 @@ func sveArrangementLetter(gOp Operation) string { func machineOpName(maskType maskShape, gOp Operation) string { asm := gOp.Asm if maskType == OneMask { - asm += "Masked" + // An SVE predicated encoding is either merging (/M) or zeroing (/Z), and + // an operation may offer only one of them; name the machine op after the + // qualifier so both can coexist and so the peepholes can tell which + // (IfElse folds into merging, Masked into zeroing). Elsewhere a mask is + // always zeroing, and keeps the historical "Masked" name. + asm += sveMaskSuffix(gOp) + if gOp.sveMergingPrefixed { + asm += "Prefixed" + } } // For ARM64, use arrangement to create distinct SSA op names if letter := sveArrangementLetter(gOp); letter != "" { @@ -361,6 +405,161 @@ func compareOperands(x, y *types.Operand) int { } } +// isInPlaceRegName reports whether an ARM register symbol names an operand that +// is written in place: , and friends, as opposed to or . +func isInPlaceRegName(name string) bool { + return len(name) >= 3 && name[1] == 'd' +} + +// sveInPlaceInput returns the index in op.In of the input naming the same +// register as the destination — the operand a destructive instruction +// overwrites — or -1 when the instruction is constructive. +// +// It fails loudly on a destination that is written in place but is not among +// the inputs, e.g. the accumulator of MLA , /M, , : that needs +// a machine op with an extra input, which simdgen does not build yet, and +// silently treating it as constructive would generate wrong code. +func (op Operation) sveInPlaceInput() int { + if len(op.Out) != 1 || op.Out[0].RegName == nil { + return -1 + } + dst := *op.Out[0].RegName + for i := range op.In { + if op.In[i].RegName != nil && *op.In[i].RegName == dst { + return i + } + } + if isInPlaceRegName(dst) { + panic(fmt.Errorf("simdgen: %s writes %s in place but does not read it as an input; "+ + "this shape is not supported yet: %s", op.Asm, dst, op)) + } + return -1 +} + +// svePredicatedOps returns the machine-level operations implied by the +// operation's inVariant: the same operation with the governing predicate as an +// ordinary input, once per qualifier the encoding supports. The inVariant +// implies machine ops only — the API is generated from the unpredicated in/out +// — and these are what the Masked/IfElse peepholes fold into. +func (op Operation) svePredicatedOps() []Operation { + if !CurrentArch().isSVE() || len(op.InVariant) != 1 || op.InVariant[0].Predication == nil { + return nil + } + var out []Operation + for i, predicate := range op.InVariant { + if predicate.Predication == nil { + continue + } + for _, qual := range *predicate.Predication { + // "M" (merging), "Z" (zeroing), or both: an encoding that offers each + // gets a machine op for each, and only the peepholes that apply to it. + q := string(qual) + p := predicate + p.Predication = &q + pred := op + // Give every operand the symbol it has in this encoding, so the + // operation describes the instruction that will be emitted and its + // shape can be read off it the same way as an unpredicated one. + pred.In = withPredRegNames(op.In, i) + pred.Out = withPredRegNames(op.Out, i) + // An operation with an unpredicated encoding has no governing + // predicate to begin with, so the variant's is a new input. One + // without (ABS) already carries its own, hidden behind an all-true + // predicate; the variant supplies the real one in its place, rather + // than a second one. + if idx := governingInput(pred.In); idx >= 0 { + pred.In[idx] = p + } else { + pred.In = append(pred.In, p) + } + pred.InVariant = nil + pred.sortOperand() + if q == "M" && pred.sveInPlaceInput() < 0 { + // A constructive instruction names its destination separately + // from its sources, and merging predication preserves that + // destination's inactive lanes, so the value it starts out + // holding is a real operand. Without it the machine op would + // claim to write a register it in fact only partly writes. + merge := pred.Out[0] + pred.In = append([]types.Operand{merge}, pred.In...) + pred.sveMergeSourceIn0 = true + } + out = append(out, pred) + } + } + return out +} + +// sveMergingPrefixedOp returns the MOVPRFX-prefixed variant of a merging +// predicated operation, or nil when the operation cannot use one. +// +// A merging SVE instruction is destructive — it merges into its own first +// source — so on its own it can only express a select whose "else" operand is +// that same source. Prefixing MOVPRFX lifts that: given +// +// ZMOVPRFX Zx, Pg/M, Zd +// ZADD Zy, Zd, Pg/M, Zd +// +// the destination holds x+y on the active lanes and whatever it already held on +// the inactive ones, so the "else" operand can be any value. The returned +// operation carries that value as an extra leading input, which makes it the +// operand the destination must share a register with (resultInArg0) and gives +// the operation a three-vreg register shape of its own. +// +// It is offered only for a commutative operation. The prefixed instruction must +// not name the destination in any operand position other than the destructive +// one, i.e. the ZADD above needs Zy != Zd; a commutative operation can always +// satisfy that by swapping its two sources, and a non-commutative one cannot. +func (op Operation) sveMergingPrefixedOp() *Operation { + if !CurrentArch().isSVE() || !op.Commutative || op.sveInPlaceInput() != 0 { + return nil + } + if len(op.Out) != 1 || op.Out[0].RegName == nil { + return nil + } + if sveMaskSuffix(op) != "Merging" { + return nil + } + // The extra input is the destination read before the operation, so it takes + // the destination's symbol; the source it displaces becomes the MOVPRFX's + // Zn, which is the symbol that instruction gives it. + merge := op.Out[0] + prefixed := op + prefixed.In = make([]types.Operand, 0, len(op.In)+1) + prefixed.In = append(prefixed.In, merge) + prefixed.In = append(prefixed.In, op.In...) + movprfxSrc := "Zn" + prefixed.In[1].RegName = &movprfxSrc + prefixed.sveMergingPrefixed = true + prefixed.sveMergeSourceIn0 = true + return &prefixed +} + +// governingInput returns the index of the governing predicate in ops, or -1 +// when there is none. +func governingInput(ops []types.Operand) int { + for i := range ops { + if ops[i].IsGoverning() { + return i + } + } + return -1 +} + +// withPredRegNames copies operands with each one's register symbol replaced by +// the symbol it has in predicated encoding i, where it has one. +func withPredRegNames(ops []types.Operand, i int) []types.Operand { + out := make([]types.Operand, len(ops)) + copy(out, ops) + for j := range out { + if names := out[j].PredRegName; names != nil && i < len(*names) { + name := (*names)[i] + out[j].RegName = &name + } + } + return out +} + // implicitPredCount reports whether the op has an implicit-all-true governing // predicate input, as a count (0 or 1). An instruction has at most one governing // predicate — the single mask input carrying a /Z or /M qualifier (see the @@ -372,7 +571,7 @@ func compareOperands(x, y *types.Operand) int { func (op Operation) implicitPredCount() int { n := 0 for i := range op.In { - if op.In[i].IsImplicitAllTrue() { + if op.In[i].IsGoverning() { n++ } } diff --git a/src/simd/archsimd/_gen/simdgen/sve/emit.go b/src/simd/archsimd/_gen/simdgen/sve/emit.go index 877667c755f779..5c0fa294819583 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/emit.go +++ b/src/simd/archsimd/_gen/simdgen/sve/emit.go @@ -7,6 +7,7 @@ package sve import ( "cmp" "fmt" + "log" "slices" "strings" @@ -37,6 +38,10 @@ func asComment(text string, width int) string { return strings.Join(lines, "\n") } +// mixedWidthLogged dedupes the mixed-element-width warning by mnemonic, so a +// conversion family with many encodings logs once per generate run. +var mixedWidthLogged = map[string]bool{} + // emit renders an operand as a unify value. Z-vectors and predicates are // scalable (a base type and per-operand element width, no fixed bits/lanes); // mem, immediate and special operands are opaque (class and position only). @@ -68,31 +73,54 @@ func (op *Operand) emit() *unify.Value { // instructions support only one; this records which. db.Add("predication", unify.NewValue(unify.NewStringExact(op.Predication))) } - if op.role == "mask" { - // role "mask" is precisely the governing predicate: the operand named - // (buildOperandList assigns the role; every instruction has at most one). It - // is implicit-all-true — dropped from the unpredicated Go API and - // synthesized as an all-true predicate at lowering, so predicated-only - // instructions (e.g. ZCMPGT) expose an unpredicated API. Flagging it here, - // not in the user's go_*.yaml, keeps the YAML unpredicated. - // - // The governing predicate is identified by name, not by a /Z or /M - // qualifier: most data-processing ops write /Z or /M, but some - // governing predicates have no qualifier (e.g. the store ST1B {.B}, - // , [...]). Either way it is . Source predicates / (e.g. in - // AND .B, /Z, .B, .B) are ordinary numbered inputs (role - // "opN"), real data, and are never flagged all-true. - db.Add("implicitAllTrue", unify.NewValue(unify.NewStringExact("true"))) + if op.governing { + // This operand is a governing predicate. + db.Add("governing", unify.NewValue(unify.NewStringExact("true"))) } if op.isList { // This register came from a single-register list ("{ . }"), a // distinct assembler encoding from a bare register. db.Add("listNumber", unify.NewValue(unify.NewStringExact("0"))) } + if op.regName != "" { + // The assembly template's register symbol, e.g. "Zdn", "Zn", "Pg". + db.Add("regName", unify.NewValue(unify.NewStringExact(op.regName))) + } + // The symbol this operand has in each predicated encoding, indexed to + // match the def's inVariant. The symbols can differ from the unpredicated + // ones to predicated ones: + // ADD , , unpredicated + // ADD , /M, , predicated + // + // [groupPredicationForms] folds the two into one def. + // simdgen needs these symbols to recognize resultInArg0. + names := make([]*unify.Value, len(op.predRegName)) + for i, n := range op.predRegName { + names[i] = unify.NewValue(unify.NewStringExact(n)) + } + db.Add("predRegName", unify.NewValue(unify.NewTuple(names...))) db.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(op.AsmPos)))) return unify.NewValue(db.Build()) } +// pickRegNames returns operand idx's symbol in each predicated encoding, in +// variant order. The encodings passed [sameOperandShape], so idx addresses the +// matching operand in every one of them. +func pickRegNames(variants []predVariant, idx int, sel func(predVariant) []string) []string { + if len(variants) == 0 { + return nil + } + out := make([]string, len(variants)) + for i, pv := range variants { + names := sel(pv) + if idx >= len(names) { + panic(fmt.Sprintf("operand %d has no counterpart in predicated encoding %d", idx, i)) + } + out[i] = names[idx] + } + return out +} + // emitOne emits a single instruction def from a fully-instantiated operand list: // the destination is the output, every other operand (including a governing // predicate) is a literal input. @@ -108,11 +136,26 @@ func (inst *Instruction) emitOne(asm string, ops []Operand) *unify.Value { db.Add("details", unify.NewValue(unify.NewStringExact(asComment(doc, 80)))) } + // One def can describe several encodings of one operation, grouped by + // [groupPredicationForms] or [groupPredicatedOnly], so each operand also + // carries the symbol it has in each predicated encoding. The symbols are + // matched up in template order, so they must be attached before the sort + // below reorders the inputs. var inOps, outOps []Operand + var outIdx, inIdx int for _, op := range ops { - if op.role == "destination" { + switch { + case op.governing: + // The governing predicate is the operand the paired encodings differ in, so + // it is not one of the symbols they are matched up by. + inOps = append(inOps, op) + case op.role == "destination": + op.predRegName = pickRegNames(inst.predVariants, outIdx, func(pv predVariant) []string { return pv.outRegNames }) + outIdx++ outOps = append(outOps, op) - } else { + default: + op.predRegName = pickRegNames(inst.predVariants, inIdx, func(pv predVariant) []string { return pv.inRegNames }) + inIdx++ inOps = append(inOps, op) } } @@ -134,7 +177,17 @@ func (inst *Instruction) emitOne(asm string, ops []Operand) *unify.Value { outs = append(outs, outOps[i].emit()) } db.Add("in", unify.NewValue(unify.NewTuple(ins...))) - db.Add("inVariant", unify.NewValue(unify.NewTuple())) + var inVar []*unify.Value + for _, pv := range inst.predVariants { + // The governing predicate of the paired predicated encoding. + var pdb unify.DefBuilder + pdb.Add("class", unify.NewValue(unify.NewStringExact("mask"))) + pdb.Add("bits", unify.NewValue(unify.NewStringExact("scalable"))) + pdb.Add("predication", unify.NewValue(unify.NewStringExact(pv.quals))) + pdb.Add("asmPos", unify.NewValue(unify.NewStringExact(fmt.Sprint(pv.predAsmPos)))) + inVar = append(inVar, unify.NewValue(pdb.Build())) + } + db.Add("inVariant", unify.NewValue(unify.NewTuple(inVar...))) db.Add("out", unify.NewValue(unify.NewTuple(outs...))) return unify.NewValue(db.Build()) } @@ -232,10 +285,34 @@ func (inst *Instruction) emitVariants(template []Operand) []*unify.Value { for _, pred := range preds { variant := make([]Operand, len(ops)) copy(variant, ops) + elem := 0 + mixedWidths := false for i := range variant { - if variant[i].Class == "mask" && variant[i].role == "mask" { + if variant[i].Class == "vreg" && variant[i].ElemBits > 0 { + if elem == 0 { + elem = variant[i].ElemBits + } else if variant[i].ElemBits != elem { + mixedWidths = true + } + } + } + for i := range variant { + if variant[i].Class != "mask" { + continue + } + if variant[i].governing { variant[i].Predication = pred } + if variant[i].ElemBits == 0 { + // This predicate doesn't come with an arrangement (which is usual). + // Get it from its peer data operand. + if mixedWidths && !mixedWidthLogged[inst.mnemonic()] { + mixedWidthLogged[inst.mnemonic()] = true + log.Printf("sve: %s: operands have mixed element widths; predicate width provisionally %d — derive esize from the pseudocode before generating an API from this def", + inst.mnemonic(), elem) + } + variant[i].ElemBits = elem + } } defs = append(defs, inst.emitOne(asm, variant)) } diff --git a/src/simd/archsimd/_gen/simdgen/sve/instruction.go b/src/simd/archsimd/_gen/simdgen/sve/instruction.go index 3cc310f4fd482c..bfa4bf6c71f651 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/instruction.go +++ b/src/simd/archsimd/_gen/simdgen/sve/instruction.go @@ -65,6 +65,32 @@ type Instruction struct { // If nil, the first iclass is used. iclass *xmlspec.Iclass mnemonicCache string + // predVariants is set on the unpredicated instruction of a + // predicated/unpredicated pair (see [groupPredicationForms]), one entry per + // predicated machine op the pair implies. It is nil for an instruction that + // comes in one form only. + predVariants []predVariant +} + +// predVariant is one predicated encoding of an operation, as seen from its +// unpredicated sibling: the governing-predicate qualifiers it offers ("M", "Z", +// or "MZ" for an encoding written /, which supports either) and its +// register symbols, in the same order as the sibling's own results and +// non-predicate inputs. +// +// One encoding can imply several machine ops — one per qualifier — but they +// share these symbols, because they are the same encoding. A second entry would +// mean a genuinely separate predicated encoding, which no paired operation in +// the ISA has today; the list exists so that such an encoding could be +// described with its own symbols rather than collapsed onto the first one's. +type predVariant struct { + quals string + outRegNames []string + inRegNames []string + // predAsmPos is the assembly position of the encoding's governing + // predicate: 1 on every encoding grouped today, but recorded rather than + // assumed — PTEST, with no destination, governs from position 0. + predAsmPos int } // ic returns the iclass this logical instruction represents, defaulting to the @@ -286,6 +312,27 @@ func (inst *Instruction) findExplanation(link string) *xmlspec.Explanation { return nil } +// symbolIsGoverning reports whether this instruction's explanation for +// register symbol name (e.g. "Pg") describes it as the governing predicate — +// the spec writes "the governing scalable predicate register" for exactly the +// symbols with that role. found reports whether any explanation names the +// symbol at all. This is the authoritative classification; [buildOperandList] +// cross-checks it against the syntactic /qualifier signal. +func (inst *Instruction) symbolIsGoverning(name string) (governing, found bool) { + want := "<" + name + ">" + for i := range inst.Explanations.Explanations { + e := &inst.Explanations.Explanations[i] + if strings.TrimSpace(e.Symbol.Value) != want { + continue + } + found = true + if strings.Contains(strings.ToLower(e.Account.Intro), "governing") { + return true, true + } + } + return false, found +} + // arngRow is one row of an arrangement size table: the encoding value of the // size field and the resulting element width in bits. type arngRow struct { @@ -399,7 +446,17 @@ func (inst *Instruction) allEncodingOperands() [][]Operand { continue } seen[s] = true - if ops := operandsFromTextA(enc.AsmTemplate.TextA); len(ops) > 0 { + ops := func() []Operand { + // A classification panic names only the operand; add which + // instruction and template it came from. + defer func() { + if r := recover(); r != nil { + panic(fmt.Sprintf("%v\n in %q template %q", r, inst.Title, s)) + } + }() + return operandsFromTextA(enc.AsmTemplate.TextA, inst.symbolIsGoverning) + }() + if len(ops) > 0 { inst.fixMemoryDirection(ops) out = append(out, ops) } @@ -454,7 +511,7 @@ func hasClass(ops []Operand, class string) bool { // bit, or a single no-op pass when the template has no governing predicate. func predicationVariants(ops []Operand) []string { for i := range ops { - if ops[i].Class == "mask" && ops[i].role == "mask" { + if ops[i].governing { if ops[i].Predication == "MZ" { return []string{"M", "Z"} } @@ -464,6 +521,44 @@ func predicationVariants(ops []Operand) []string { return []string{""} } +// predicationForm reports whether this encoding is the predicated or the +// unpredicated form of an operation, as "predicated" / "unpredicated". +// +// It reads the encoding rather than the title: an encoding that takes a +// governing predicate is the predicated one. SVE does also spell this out in +// the title of an operation that has both forms ("ADD (vectors, predicated)" +// and "ADD (vectors, unpredicated)"), and [predicationGroupKey] uses that to pair +// them, but an operation that only comes predicated says nothing in its title — +// both of ABS's encodings are titled plain "ABS". +func (inst *Instruction) predicationForm() string { + for _, ops := range inst.allEncodingOperands() { + for i := range ops { + if ops[i].governing { + return "predicated" + } + } + } + return "unpredicated" +} + +// predicationGroupKey returns the key that groups the encodings of one +// operation: the title with any predicated/unpredicated qualifier removed, e.g. +// both "ADD (vectors, predicated)" and "ADD (vectors, unpredicated)" yield "add +// (vectors)", and both of ABS's encodings yield "abs". +// +// Encodings that are not variations on one another keep distinct titles — "ADD +// (immediate)", "ADD (extended register)" — so they land in groups of their own, +// which groupPredicationForms then leaves alone. +func (inst *Instruction) predicationGroupKey() string { + t := strings.ToLower(inst.Title) + t = strings.ReplaceAll(t, "unpredicated", "") + t = strings.ReplaceAll(t, "predicated", "") + // Tidy the separator the qualifier left behind: "(vectors, )" -> "(vectors)". + t = strings.ReplaceAll(t, ", )", ")") + t = strings.ReplaceAll(t, "( ", "(") + return strings.Join(strings.Fields(t), " ") +} + // documentation returns a one-line description of the instruction. func (inst *Instruction) documentation() string { if len(inst.Desc.Authored.Paragraphs) > 0 { diff --git a/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go b/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go index 530f3d82660bde..9dd7372baae645 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go +++ b/src/simd/archsimd/_gen/simdgen/sve/instruction_test.go @@ -214,9 +214,9 @@ func TestOperandsPredicated(t *testing.T) { ops := parse(t, addPred).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } - want := []string{"vreg:destination", "mask:mask", "vreg:op0", "vreg:op1"} + want := []string{"vreg:destination", "mask:governing", "vreg:op0", "vreg:op1"} if !reflect.DeepEqual(got, want) { t.Errorf("predicated operands = %v, want %v", got, want) } @@ -423,10 +423,10 @@ func TestReductionOutput(t *testing.T) { ops := parse(t, saddv).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } // The scalar result
is a SIMD&FP register destination, not an input. - want := []string{"vreg:destination", "mask:mask", "vreg:op0"} + want := []string{"vreg:destination", "mask:governing", "vreg:op0"} if !reflect.DeepEqual(got, want) { t.Errorf("SADDV operands = %v, want %v", got, want) } @@ -474,13 +474,13 @@ func TestStoreReglist(t *testing.T) { ops := parse(t, st1b).operands() var got []string for _, op := range ops { - got = append(got, op.Class+":"+op.role) + got = append(got, op.Class+":"+opRole(op)) } // The single-register list unwraps to a vreg (the data source); the memory // operand is the store destination. Order follows the source template. The // predicate is : a governing predicate (role "mask"), even though a // store writes no /Z or /M qualifier. - want := []string{"vreg:op0", "mask:mask", "mem:destination"} + want := []string{"vreg:op0", "mask:governing", "mem:destination"} if !reflect.DeepEqual(got, want) { t.Errorf("ST1B operands = %v, want %v", got, want) } @@ -522,3 +522,119 @@ func TestMemoryOperandClassified(t *testing.T) { } } } + +// opRole renders an operand's partition for test expectations: its role, or +// "governing" for the governing predicate, which has no numbered role. +func opRole(op Operand) string { + if op.governing { + return "governing" + } + return op.role +} + +// absExplanations is the explanation block for the ABS fixtures: the size table +// plus a explanation using the spec's "governing scalable predicate +// register" wording, so parsing exercises the explanation-driven governing +// classification rather than the syntactic fallback. +const absExplanations = ` + + + <T> + + + B + H + S + D +
+ + + + <Pg> + Is the name of the governing scalable predicate register, encoded in the "Pg" field. + + ` + +// absSection builds one encoding of ABS, a predicated-only operation: both +// encodings are titled plain "ABS" (nothing in the title says "predicated"), +// and they differ only in the governing predicate's qualifier. +func absSection(id, qual string) string { + return ` + + + + + Absolute value of the signed integer in each active element. + + ABS <Zd>.<T>, <Pg>/` + qual + `, <Zn>.<T> + ` + absExplanations + `` +} + +// TestGroupPredicatedOnly covers the emission path of an operation with no +// unpredicated encoding at all: the merging encoding carries the operation, +// keeping its governing predicate as an implicit-all-true input, and the group +// becomes an inVariant on it; the sibling encoding is covered and not emitted. +func TestGroupPredicatedOnly(t *testing.T) { + m := parse(t, absSection("abs_m", "M")) + z := parse(t, absSection("abs_z", "Z")) + covered := groupPredicationForms([]*Instruction{m, z}) + if covered[m] || !covered[z] { + t.Fatalf("covered[m]=%v covered[z]=%v, want the merging carrier kept and the zeroing sibling covered", covered[m], covered[z]) + } + if len(m.predVariants) != 1 || m.predVariants[0].quals != "M" { + t.Fatalf("carrier predVariants = %+v, want one variant with quals M", m.predVariants) + } + + defs := m.emitAll() + if len(defs) == 0 { + t.Fatal("carrier emitAll returned no defs") + } + for _, d := range defs { + var op struct { + In []struct { + Class string + Predication *string + Governing *bool + RegName *string `unify:"regName"` + PredRegName *[]string `unify:"predRegName"` + } `unify:"in"` + InVariant []struct { + Class string + Predication *string + } `unify:"inVariant"` + Out []struct { + PredRegName *[]string `unify:"predRegName"` + } `unify:"out"` + } + if err := d.Decode(&op); err != nil { + t.Fatal(err) + } + var sawGoverning, sawVreg bool + for _, in := range op.In { + switch in.Class { + case "mask": + if in.Governing == nil || !*in.Governing { + t.Errorf("mask input not marked governing: %+v", in) + } + if in.Predication == nil || *in.Predication != "M" { + t.Errorf("governing predicate predication = %v, want M", in.Predication) + } + sawGoverning = true + case "vreg": + if in.PredRegName == nil || !reflect.DeepEqual(*in.PredRegName, []string{"Zn"}) { + t.Errorf("vreg input predRegName = %v, want [Zn]", in.PredRegName) + } + sawVreg = true + } + } + if !sawGoverning || !sawVreg { + t.Errorf("inputs missing governing mask or vreg: %+v", op.In) + } + if len(op.InVariant) != 1 || op.InVariant[0].Predication == nil || *op.InVariant[0].Predication != "M" { + t.Errorf("inVariant = %+v, want one mask with predication M", op.InVariant) + } + if len(op.Out) != 1 || op.Out[0].PredRegName == nil || !reflect.DeepEqual(*op.Out[0].PredRegName, []string{"Zd"}) { + t.Errorf("out predRegName = %+v, want [Zd]", op.Out) + } + } +} diff --git a/src/simd/archsimd/_gen/simdgen/sve/load.go b/src/simd/archsimd/_gen/simdgen/sve/load.go index 45d367b41af3a7..9b16deaa3be321 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/load.go +++ b/src/simd/archsimd/_gen/simdgen/sve/load.go @@ -5,6 +5,7 @@ package sve import ( + "slices" "sort" "simd/archsimd/_gen/unify" @@ -50,9 +51,197 @@ func Load(path string) ([]*unify.Value, error) { if err != nil { return nil, err } + covered := groupPredicationForms(insts) var defs []*unify.Value for _, inst := range insts { + if covered[inst] { + // The predicated half of a pair; it is emitted as an inVariant of its + // unpredicated sibling so the operation has a single unifier value. + continue + } defs = append(defs, inst.emitAll()...) } return defs, nil } + +// groupPredicationForms pairs the predicated and unpredicated encodings of the +// same operation and folds them into one definition, mirroring how the AMD64 +// loader treats an AVX-512 instruction's optional K-mask: the unpredicated form +// supplies the operation (and therefore the single front-end API), and the +// governing predicate becomes an inVariant that simdgen turns into predicated +// machine ops plus peepholes. +// However, different from AVX-512, where the predication mode is orthogonal to +// the operation as an instruction suffix, SVE's predication modes are separate +// instruction encodings, so the loader has to pair them up. +// +// A pair is only formed when both forms actually exist and their operand shapes +// correspond; the returned set names the predicated instructions that the pair +// covers, which the caller then skips. Everything else — an operation with only +// a predicated form (whose predicate stays implicit-all-true), or only an +// unpredicated one — is emitted unchanged. +func groupPredicationForms(insts []*Instruction) map[*Instruction]bool { + type group struct{ unpred, pred []*Instruction } + groups := map[string]*group{} + for _, inst := range insts { + key := inst.predicationGroupKey() + if key == "" { + continue + } + g := groups[key] + if g == nil { + g = &group{} + groups[key] = g + } + if inst.predicationForm() == "unpredicated" { + g.unpred = append(g.unpred, inst) + } else { + g.pred = append(g.pred, inst) + } + } + + covered := map[*Instruction]bool{} + for _, g := range groups { + if len(g.unpred) == 0 && len(g.pred) > 1 { + groupPredicatedOnly(g.pred, covered) + continue + } + if len(g.unpred) != 1 || len(g.pred) == 0 { + // Not a clean pair (a form is missing, or the title is ambiguous); + // leave both halves to be emitted as they are. + continue + } + un := g.unpred[0] + unOps := un.operands() + var variants []predVariant + for _, pr := range g.pred { + prOps := pr.operands() + if !sameOperandShape(unOps, prOps) { + continue + } + var quals string + for _, q := range predicationVariants(prOps) { + quals += q + } + if quals == "" { + continue + } + // Each encoding carries its own register symbols, so a machine op is + // always generated from the shape of the encoding it comes from. + outs, ins := splitRegNames(prOps) + variants = append(variants, predVariant{quals: quals, outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(prOps)}) + covered[pr] = true + } + if len(variants) == 0 { + continue + } + un.predVariants = variants + } + return covered +} + +// groupPredicatedOnly folds the encodings of an operation that has no +// unpredicated form at all — SVE writes ABS as "ABS ., /M, ." +// and "ABS ., /Z, .", and nothing else. +// +// There is no unpredicated encoding to carry the operation, so one of the +// predicated encodings does. Its governing predicate stays implicit-all-true, so +// the front-end API is still unpredicated, and every qualifier in the group — +// its own included — becomes an inVariant qualifier, which simdgen turns into +// one predicated machine op each for the peepholes to fold into. +// +// Only the merging encoding is used. Zeroing predication on these instructions +// is an Armv9.6-A extension -- ABS assembles to a different opcode under /Z, and +// baseline SVE hardware traps it -- while merging is available wherever SVE is. +// Nothing in what the XML parser exposes tells the two apart: both carry +// instr-class "sve", and the arch_variant element that does record the +// difference is not surfaced. So the zeroing encodings are dropped here rather +// than gated, to be folded in with the rest of SVE2.2 once simdgen can gate on +// the SVE sub-level. +// +// The group is folded only when the encodings are variations on one predication +// mode and nothing else: same operand shape, and one encoding per qualifier. Two +// encodings sharing a qualifier are two different instructions that happen to +// share a title (addressing modes of a load, say), and are left alone. +func groupPredicatedOnly(pred []*Instruction, covered map[*Instruction]bool) { + byQual := map[string]*Instruction{} + shape := pred[0].operands() + for _, inst := range pred { + ops := inst.operands() + if !sameOperandShape(shape, ops) { + return + } + quals := predicationVariants(ops) + if len(quals) != 1 || quals[0] == "" { + return + } + if _, dup := byQual[quals[0]]; dup { + return + } + byQual[quals[0]] = inst + } + base, ok := byQual["M"] + if !ok { + return + } + baseOps := base.operands() + outs, ins := splitRegNames(baseOps) + base.predVariants = []predVariant{{quals: "M", outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(baseOps)}} + for _, inst := range byQual { + if inst != base { + covered[inst] = true + } + } +} + +// governingAsmPos returns the assembly position of the governing predicate in +// ops. Both callers work on encodings already classified as predicated, so a +// missing governing predicate is a broken invariant, not a case. +func governingAsmPos(ops []Operand) int { + for i := range ops { + if ops[i].governing { + return ops[i].AsmPos + } + } + panic("sve: predicated encoding has no governing predicate") +} + +// splitRegNames returns an operand template's register symbols, results first +// and then the non-predicate inputs, in the order sameOperandShape compares +// them, so the two halves of a pair line up element by element. +func splitRegNames(ops []Operand) (outs, ins []string) { + for i := range ops { + if ops[i].governing { + continue + } + if ops[i].role == "destination" { + outs = append(outs, ops[i].regName) + } else { + ins = append(ins, ops[i].regName) + } + } + return outs, ins +} + +// sameOperandShape reports whether two operand templates describe the same +// operation apart from a governing predicate: same result and same sequence of +// non-predicate input classes. The predicated form of a destructive operation +// names its destination twice (once as the in-place source), which +// buildOperandList already turns into a regular input, so the two shapes line up. +func sameOperandShape(a, b []Operand) bool { + split := func(ops []Operand) (outs, ins []string) { + for i := range ops { + if ops[i].governing { + continue // the governing predicate is what differs + } + if ops[i].role == "destination" { + outs = append(outs, ops[i].Class) + } else { + ins = append(ins, ops[i].Class) + } + } + return outs, ins + } + ao, ai := split(a) + bo, bi := split(b) + return slices.Equal(ao, bo) && slices.Equal(ai, bi) +} diff --git a/src/simd/archsimd/_gen/simdgen/sve/operands.go b/src/simd/archsimd/_gen/simdgen/sve/operands.go index 52b2f4b11e1b15..bc905a48ba95da 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/operands.go +++ b/src/simd/archsimd/_gen/simdgen/sve/operands.go @@ -6,6 +6,7 @@ package sve import ( "fmt" + "log" "regexp" "strings" @@ -109,11 +110,16 @@ type Operand struct { // and unknown operands so diagnostics can name what was skipped. Raw string - // role is the operand's internal role: "destination", "op0"/"op1"/..., or - // "mask" (a governing predicate). It drives out/in/inVariant partitioning at - // emit time but is NOT emitted (simdgen orders operands by AsmPos, so a role - // field in the YAML would be redundant). + // role is the operand's internal role: "destination" or "op0"/"op1"/.... + // It drives out/in partitioning at emit time but is NOT emitted (simdgen + // orders operands by AsmPos, so a role field in the YAML would be + // redundant). A governing predicate has no role; it is marked by governing. role string + // governing marks the governing predicate — the operand selecting which + // lanes the instruction acts on, as opposed to a predicate read as data. + // It is classified from the spec's own explanation text for the symbol and + // emitted as the def's "governing" field. + governing bool // arngLink is the link of this operand's arrangement symbol (// // ), used to resolve its per-operand element widths. Empty if the // operand has a fixed or no arrangement. @@ -130,6 +136,11 @@ type Operand struct { isList bool // regName is the inner register symbol, e.g. "Zdn", "Zm", "Pg". regName string + // predRegName is the symbol this operand has in each paired predicated + // encoding, indexed to match the operation's predicated variants (and so the + // inVariant tuple the def carries). It is nil when the operation has no + // predicated form. + predRegName []string } // resultInArg0 reports whether this destination register is also read, i.e. it @@ -215,17 +226,20 @@ type tok struct { } // operandsFromTextA parses operands from an assembly template's / -// sequence, preserving each operand's arrangement-symbol link. -func operandsFromTextA(textA []xmlspec.TextA) []Operand { - return buildOperandList(classifyToks(tokenizeTextA(textA))) +// sequence, preserving each operand's arrangement-symbol link. govern reports +// whether a predicate register symbol is the governing predicate; the real +// loader path passes the instruction's explanation lookup. +func operandsFromTextA(textA []xmlspec.TextA, govern func(regName string) (governing, found bool)) []Operand { + return buildOperandList(classifyToks(tokenizeTextA(textA)), govern) } // operands parses operands from a flattened template string. It cannot recover -// links, so arrangement symbols resolve to empty links; it is used for +// links or explanations, so arrangement symbols resolve to empty links and +// the governing predicate is classified syntactically; it is used for // classification-only paths and tests. The real loader path uses // operandsFromTextA. func operands(asmTemplate string) []Operand { - return buildOperandList(classifyToks(tokenizeString(asmTemplate))) + return buildOperandList(classifyToks(tokenizeString(asmTemplate)), nil) } // tokenizeTextA splits a / sequence into operand tokens on top-level @@ -493,9 +507,9 @@ func isInPlaceReg(name string) bool { // // Unlike an AMD64 AVX-512 K-mask, an SVE governing predicate is NOT optional: // there is no K0-style "no predicate" encoding, so it is a mandatory literal -// input (class "mask", role "mask"), not an inVariant. See the discussion in -// emitOne. -func buildOperandList(parsed []tok) []Operand { +// input (class "mask", marked governing), not an inVariant. See the discussion +// in emitOne. +func buildOperandList(parsed []tok, govern func(regName string) (governing, found bool)) []Operand { var outs, ins []Operand inputCount := 0 destAssigned := false @@ -534,16 +548,49 @@ func buildOperandList(parsed []tok) []Operand { } switch p.operandType { case OperandPReg: - if p.regName == "Pg" || p.predication != "" { - // Governing predicate: the operand named ("g" for governing), a - // mandatory mask input (role "mask", not a numbered opN). Most carry a - // /Z or /M qualifier (predicated data-processing ops), but some do not - // — e.g. the store ST1B {.B}, , [...] governs with a plain - // — so key on the register name, not the qualifier. Source - // predicates / and the destination are ordinary operands, - // filed by place() below. + // The governing predicate is classified from the spec's own words: + // its explanation calls the symbol "the governing scalable predicate + // register". The syntactic signal — the symbol is , or it carries + // a /M or /Z qualifier — is kept as a cross-check, so a shape where + // the two diverge fails loudly instead of misclassifying: the SME + // outer products govern with two predicates spelled / + // (SUMOPA .S, /M, /M, .B, .B), which does not + // fit the one-governing-predicate shape simdgen builds on and must + // be rejected here, not silently halved. The bare-string parse path + // (tests, diagnostics) has no explanations and uses the syntactic + // signal alone. + syntactic := p.regName == "Pg" || p.predication != "" + governing := syntactic + if govern != nil { + if verdict, found := govern(p.regName); found { + governing = verdict + if governing != syntactic { + // The explanation wins, but say so. Two shapes diverge in + // the ISA today, in opposite directions: the MOV alias of + // SEL (MOV ., /M, .), whose keeps + // its "select" description from SEL even though the alias + // writes it with a qualifier; and the bare of the + // predicate-as-counter loads/stores, which the spec calls + // governing though it is neither nor qualified. + // Neither instruction is emitted today, so nothing + // downstream sees the difference — but the second is the + // case the explanation gets right and the syntax cannot. + log.Printf("sve: symbol <%s> (qualifier %q): explanation says governing=%v, syntactic signal says %v; following the explanation", + p.regName, p.predication, governing, syntactic) + } + } + // No explanation for this symbol — an alias template, or a test + // fixture with the explanations stripped — so the syntactic + // signal stands alone. + } + if governing { + // A mandatory mask input, not a numbered opN. Most carry a /Z or + // /M qualifier (predicated data-processing ops), but some do not + // — the store ST1B {.B}, , [...] governs with a plain + // . Source predicates / and the destination are + // ordinary operands, filed by place() below. ins = append(ins, Operand{ - Type: OperandPReg, Class: "mask", role: "mask", + Type: OperandPReg, Class: "mask", governing: true, Predication: p.predication, AsmPos: p.asmPos, arngLink: p.arngLink, fixedElem: p.fixedElem, regName: p.regName, }) @@ -571,6 +618,19 @@ func buildOperandList(parsed []tok) []Operand { }, p.isDestination) } } + // An instruction has at most one governing predicate — everything simdgen + // derives from the field (implicitPredCount, regShape, the all-true + // synthesis) assumes it. The SME outer products break this (SUMOPA governs + // with /M and /M at once) and must be rejected here, not halved. + governCount := 0 + for i := range ins { + if ins[i].governing { + governCount++ + } + } + if governCount > 1 { + panic(fmt.Sprintf("sve: %d governing predicates in one operand list; only one is supported", governCount)) + } return append(outs, ins...) } diff --git a/src/simd/archsimd/_gen/simdgen/types/operation.go b/src/simd/archsimd/_gen/simdgen/types/operation.go index cdadfbc2fa8ae1..35476f7259570f 100644 --- a/src/simd/archsimd/_gen/simdgen/types/operation.go +++ b/src/simd/archsimd/_gen/simdgen/types/operation.go @@ -100,12 +100,38 @@ type Operand struct { // Currently only list number 0 is supported (we might need to teach regalloc handle register lists // to support more than one register in the list). ListNumber *int - // ImplicitAllTrue marks an SVE governing-predicate input that is dropped from - // the user-facing (unpredicated) API: the generated method/generic op/intrinsic - // omit it, and the lowering synthesizes an all-true predicate for it. This is - // how predicated-only SVE instructions (e.g. ZCMPGT) expose an unpredicated Go - // API, for #79781. - ImplicitAllTrue *bool + // RegName is the assembly template's register symbol for this operand, e.g. + // "Zdn", "Zn", "Pg" (SVE only). Comparing it across operands is how the + // shape of an instruction is recognised: an input naming the same register + // as the destination is written in place. + RegName *string + // PredRegName is the symbol this operand has in each of the operation's + // predicated encodings, indexed to match InVariant (SVE only). It is nil + // for an operation with no predicated encoding, and for every other target. + PredRegName *[]string + // Predication is the SVE governing-predicate qualifier, "M" (merging) or + // "Z" (zeroing). It is set on mask operands of predicated encodings and + // decides whether the generated machine op is the merging or the zeroing + // form (see sveMaskSuffix). + Predication *string + // Governing marks the SVE governing predicate among an instruction's + // operands — the one that selects which lanes the instruction acts on, as + // opposed to a predicate it merely reads as data (SEL's , the / + // of a predicate-logical op). + // + // It is set only where the instruction has no unpredicated encoding, since + // otherwise that encoding carries the operation and its predicated sibling's + // predicate becomes an InVariant instead. So a governing predicate here is + // always one the Go API hides: the generated method, generic op and + // intrinsic omit it, and the lowering synthesizes an all-true predicate in + // its place, which is how predicated-only instructions (ZCMPGT) expose an + // unpredicated API. See #79781. + // + // This is independent of [Operand.Predication]: a governing predicate need + // not carry a qualifier (SADDV
, , . has no lanes to merge + // into), and a qualified predicate need not be governing in this sense (the + // InVariant of a paired operation is a real operand a peephole supplies). + Governing *bool } // VectorSize is a unifier value that is either a number or the string "scalable". @@ -136,10 +162,10 @@ func (vs VectorSize) String() string { return fmt.Sprint(vs.N()) } -// IsImplicitAllTrue reports whether this operand is an SVE governing predicate -// that is dropped from the API and filled with an all-true predicate at lowering. -func (o *Operand) IsImplicitAllTrue() bool { - return o.ImplicitAllTrue != nil && *o.ImplicitAllTrue +// IsGoverning reports whether this operand is the SVE governing predicate, and +// so is dropped from the API and filled with an all-true predicate at lowering. +func (o *Operand) IsGoverning() bool { + return o.Governing != nil && *o.Governing } func (o Operand) OpName(s string) string { diff --git a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go index 79cbe3da76a4e7..bcb2facd2c7b55 100644 --- a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go @@ -194,3 +194,108 @@ func TestStringSVE(t *testing.T) { t.Logf("mx=%s", mx) t.Logf("my=%s", my) } + +//go:noinline +func keepAliveInt8s(archsimd.Int8s) {} + +// TestIfElseSVE checks IfElse and Masked, and that the merging peephole keeps +// the same semantics whether or not it fires: x.Add(y).IfElse(m, x) folds into a +// predicated add, x.Add(y).IfElse(m, z) does not, and both must agree with a +// lane-by-lane reference. +func TestIfElseSVE(t *testing.T) { + if !archsimd.ARM64.SVE() { + t.Skip("no sve") + } + n := archsimd.Int8s{}.Len() + xs, ys, zs := make([]int8, n), make([]int8, n), make([]int8, n) + for i := range xs { + xs[i] = int8(i + 1) + ys[i] = int8(i % 3) // active where xs[i] > ys[i], which alternates early on + zs[i] = int8(-i - 1) + } + x, y, z := archsimd.LoadInt8s(xs), archsimd.LoadInt8s(ys), archsimd.LoadInt8s(zs) + m := x.Greater(y) + + got := make([]int8, n) + check := func(name string, v archsimd.Int8s, want func(i int, active bool) int8) { + t.Helper() + v.Store(got) + for i := 0; i < n; i++ { + if w := want(i, xs[i] > ys[i]); got[i] != w { + t.Errorf("%s: lane %d = %d, want %d (x=%d y=%d)", name, i, got[i], w, xs[i], ys[i]) + } + } + } + + check("IfElse", x.IfElse(m, y), func(i int, active bool) int8 { + if active { + return xs[i] + } + return ys[i] + }) + check("Masked", x.Masked(m), func(i int, active bool) int8 { + if active { + return xs[i] + } + return 0 + }) + // Folds into the merging-predicated add. + check("Add.IfElse(x)", x.Add(y).IfElse(m, x), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return xs[i] + }) + // Folds via commutativity. + check("Add.IfElse(y)", x.Add(y).IfElse(m, y), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return ys[i] + }) + // Folds behind a merging MOVPRFX: the else operand is neither source. + check("Add.IfElse(z)", x.Add(y).IfElse(m, z), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return zs[i] + }) + // ADD has no zeroing-predicated form, so Masked folds into the merging one + // with the zero vector as its else operand. + check("Add.Masked", x.Add(y).Masked(m), func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return 0 + }) + + // The prefixed path with every operand still live afterwards, so the + // destination can be none of them and the merging MOVPRFX has to place the + // else operand itself. + rz := x.Add(y).IfElse(m, z) + keepAliveInt8s(x) + keepAliveInt8s(y) + keepAliveInt8s(z) + check("Add.IfElse(z) with all live", rz, func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return zs[i] + }) + + // The MOVPRFX path: x must survive the destructive predicated add. + r := x.Add(y).IfElse(m, x) + keepAliveInt8s(x) + check("Add.IfElse(x) with x live", r, func(i int, active bool) int8 { + if active { + return xs[i] + ys[i] + } + return xs[i] + }) + x.Store(got) + for i := 0; i < n; i++ { + if got[i] != xs[i] { + t.Errorf("x clobbered by MOVPRFX: lane %d = %d, want %d", i, got[i], xs[i]) + } + } +} diff --git a/src/simd/archsimd/types_sve.go b/src/simd/archsimd/types_sve.go index 6647aab9037e7d..2c817b381161fe 100644 --- a/src/simd/archsimd/types_sve.go +++ b/src/simd/archsimd/types_sve.go @@ -82,6 +82,21 @@ func (x Float32s) StorePart(s []float32) int { //go:noescape func (x Float32s) storePart(s []float32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Float32s) IfElse(mask Mask32s, y Float32s) Float32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Float32s) Masked(mask Mask32s) Float32s { + var zero Float32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Float32s) String() string { @@ -159,6 +174,21 @@ func (x Float64s) StorePart(s []float64) int { //go:noescape func (x Float64s) storePart(s []float64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Float64s) IfElse(mask Mask64s, y Float64s) Float64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Float64s) Masked(mask Mask64s) Float64s { + var zero Float64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Float64s) String() string { @@ -236,6 +266,21 @@ func (x Int8s) StorePart(s []int8) int { //go:noescape func (x Int8s) storePart(s []int8) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int8s) IfElse(mask Mask8s, y Int8s) Int8s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int8s) Masked(mask Mask8s) Int8s { + var zero Int8s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int8s) String() string { @@ -313,6 +358,21 @@ func (x Int16s) StorePart(s []int16) int { //go:noescape func (x Int16s) storePart(s []int16) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int16s) IfElse(mask Mask16s, y Int16s) Int16s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int16s) Masked(mask Mask16s) Int16s { + var zero Int16s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int16s) String() string { @@ -390,6 +450,21 @@ func (x Int32s) StorePart(s []int32) int { //go:noescape func (x Int32s) storePart(s []int32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int32s) IfElse(mask Mask32s, y Int32s) Int32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int32s) Masked(mask Mask32s) Int32s { + var zero Int32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int32s) String() string { @@ -467,6 +542,21 @@ func (x Int64s) StorePart(s []int64) int { //go:noescape func (x Int64s) storePart(s []int64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Int64s) IfElse(mask Mask64s, y Int64s) Int64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Int64s) Masked(mask Mask64s) Int64s { + var zero Int64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Int64s) String() string { @@ -544,6 +634,21 @@ func (x Uint8s) StorePart(s []uint8) int { //go:noescape func (x Uint8s) storePart(s []uint8) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint8s) IfElse(mask Mask8s, y Uint8s) Uint8s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint8s) Masked(mask Mask8s) Uint8s { + var zero Uint8s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint8s) String() string { @@ -621,6 +726,21 @@ func (x Uint16s) StorePart(s []uint16) int { //go:noescape func (x Uint16s) storePart(s []uint16) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint16s) IfElse(mask Mask16s, y Uint16s) Uint16s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint16s) Masked(mask Mask16s) Uint16s { + var zero Uint16s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint16s) String() string { @@ -698,6 +818,21 @@ func (x Uint32s) StorePart(s []uint32) int { //go:noescape func (x Uint32s) storePart(s []uint32) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint32s) IfElse(mask Mask32s, y Uint32s) Uint32s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint32s) Masked(mask Mask32s) Uint32s { + var zero Uint32s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint32s) String() string { @@ -775,6 +910,21 @@ func (x Uint64s) StorePart(s []uint64) int { //go:noescape func (x Uint64s) storePart(s []uint64) +// IfElse returns the elements of x where the corresponding element of mask is +// true, and the elements of y where it is false. +// +// Asm: ZSEL +func (x Uint64s) IfElse(mask Mask64s, y Uint64s) Uint64s + +// Masked returns the elements of x where the corresponding element of mask is +// true, and zero where it is false. +// +// Asm: Emulated +func (x Uint64s) Masked(mask Mask64s) Uint64s { + var zero Uint64s + return x.IfElse(mask, zero) +} + // String returns a string representation of SIMD vector x. Only the x.Len() // elements that exist at the runtime vector length are shown. func (x Uint64s) String() string { diff --git a/test/codegen/simd_arm64.go b/test/codegen/simd_arm64.go index 5911a2d8b654f6..9fc1a2444e7ee8 100644 --- a/test/codegen/simd_arm64.go +++ b/test/codegen/simd_arm64.go @@ -155,9 +155,59 @@ func loToHiUint16Vec(x, lo archsimd.Uint16x8) archsimd.Uint16x8 { return x.ReshapeToUint64s().BitsToFloat64().SetElem(1, lo.ReshapeToUint64s().BitsToFloat64().GetElem(0)).ToBits().ReshapeToUint16s() } +// --- SVE predication peepholes --- +// +// IfElse over an unpredicated operation folds into that operation's +// merging-predicated form. Merging keeps the destination, and an SVE predicated +// instruction is destructive, so an "else" operand that is already one of the +// sources folds into a bare predicated instruction; any other one first needs a +// merging MOVPRFX to put it in the destination. + +func sveIfElseFoldsFirstOperand(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // arm64:`ZADD.*P[0-9]+\.M` -`ZSEL` -`ZMOVPRFX` + return x.Add(y).IfElse(m, x) +} + +func sveIfElseFoldsSecondOperand(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // Commutative, so the mirrored form folds too. + // arm64:`ZADD.*P[0-9]+\.M` -`ZSEL` -`ZMOVPRFX` + return x.Add(y).IfElse(m, y) +} + +func sveIfElseArbitraryElse(x, y, z archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // The else operand is neither source, so a merging MOVPRFX puts it in the + // destination and the destructive add merges over it. + // arm64:`ZMOVPRFX.*P[0-9]+\.M` `ZADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).IfElse(m, z) +} + +func sveMaskedFoldsIntoMerging(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // Masked is a select against zero. ADD has no zeroing-predicated form, so it + // folds into the merging one with the zero vector as the else operand. + // arm64:`ZMOVPRFX.*P[0-9]+\.M` `ZADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).Masked(m) +} + +//go:noinline +func sinkInt8s(archsimd.Int8s) {} + +func sveIfElseMovprfx(x, y archsimd.Int8s, m archsimd.Mask8s) archsimd.Int8s { + // The else operand is a source, but x stays live so the destructive add + // cannot write it. The whole register is copied, not just the active lanes, + // so this prefix is the unpredicated MOVPRFX. + // arm64:`ZMOVPRFX` `ZADD.*P[0-9]+\.M` -`ZMOVPRFX.*P[0-9]+` + r := x.Add(y).IfElse(m, x) + sinkInt8s(x) + return r +} + +func sveIfElseFloat(x, y archsimd.Float64s, m archsimd.Mask64s) archsimd.Float64s { + // arm64:`ZFADD.*P[0-9]+\.M` -`ZSEL` + return x.Add(y).IfElse(m, x) +} + // The zero value of a mask is an all-false predicate. func sveZeroMask() archsimd.Mask8s { // arm64:`PPFALSE` -`ZDUP` var m archsimd.Mask8s - return m -} + return m}