From 7f9af814e191d4d5008b9c787b8f99fffbf1dd8f Mon Sep 17 00:00:00 2001 From: Keith Randall Date: Thu, 3 Sep 2026 09:43:27 -0700 Subject: [PATCH 1/5] cmd/compile: add SizeAndAlign support to ssa check mode Forgot to add SizeAndAlign support to the ssa check mode in CL 824984. Fixes #65827 Change-Id: I8582e43bd7b2521aac0d2fa51db9ce54333bd08c Reviewed-on: https://go-review.googlesource.com/c/go/+/827324 LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Auto-Submit: Keith Randall Reviewed-by: David Chase Reviewed-by: Keith Randall Reviewed-by: Joel Sing --- src/cmd/compile/internal/ssacompile/check.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/cmd/compile/internal/ssacompile/check.go b/src/cmd/compile/internal/ssacompile/check.go index 06c6f5618997eb..fa9bc4ae2cf599 100644 --- a/src/cmd/compile/internal/ssacompile/check.go +++ b/src/cmd/compile/internal/ssacompile/check.go @@ -223,8 +223,14 @@ func checkFunc(f *ssa.Func) { case ssaop.AuxTypePanicBoundsC, ssaop.AuxTypePanicBoundsCC: canHaveAux = true canHaveAuxInt = true + case ssaop.AuxTypeSizeAndAlign: + if _, ok := v.Aux.(ssa.Int64Aux); !ok { + f.Fatalf("value %v has Aux type %T, want Int64Aux", v, v.Aux) + } + canHaveAux = true + canHaveAuxInt = true default: - f.Fatalf("unknown aux type for %s", v.Op) + f.Fatalf("unknown aux type %T for %s", ssaop.OpcodeTable[v.Op].AuxType, v.Op) } if !canHaveAux && v.Aux != nil { f.Fatalf("value %s has an Aux value %v but shouldn't", v.LongString(), v.Aux) From 70c31c1f4ff8ba16f85d4ab7787041cb072bf31e Mon Sep 17 00:00:00 2001 From: Junyang Shao Date: Thu, 3 Sep 2026 16:02:54 -0400 Subject: [PATCH 2/5] cmd/compile: include SVE predicate registers in arm64 callerSave The arm64 callerSave set omitted the sixteen P registers, so the register allocator believed calls preserve them and could keep a live mask in a P register across a call that clobbers it. amd64 includes its K mask registers in callerSave; do the same for pred. For #79781. Change-Id: I5c64e751f0d10a66615208beff9f13858cd0730a Reviewed-on: https://go-review.googlesource.com/c/go/+/827485 Reviewed-by: Junyang Shao Reviewed-by: David Chase LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Auto-Submit: Junyang Shao --- src/cmd/compile/internal/ssa/_gen/ARM64Ops.go | 2 +- src/cmd/compile/internal/ssa/ssaop/opGen.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go index f53e48bf160b06..1e8ca8bb205552 100644 --- a/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/ARM64Ops.go @@ -161,7 +161,7 @@ func init() { gpspsbg = gpspg.union(buildReg("SB")) fp = buildReg("F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31") pred = buildReg("P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15") - callerSave = gp.union(fp).union(buildReg("g")) // runtime.setg (and anything calling it) may clobber g + callerSave = gp.union(fp).union(pred).union(buildReg("g")) // runtime.setg (and anything calling it) may clobber g r25 = buildReg("R25") r24to25 = buildReg("R24 R25") f16to17 = buildReg("F16 F17") diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index f5055e6fa72772..cc2582229d5f55 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -80243,7 +80243,7 @@ var OpcodeTable = [...]OpInfo{ ClobberFlags: true, Call: true, Reg: RegInfo{ - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 }, }, { @@ -80254,7 +80254,7 @@ var OpcodeTable = [...]OpInfo{ Call: true, tailCall: true, Reg: RegInfo{ - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 }, }, { @@ -80268,7 +80268,7 @@ var OpcodeTable = [...]OpInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 335544319, V2: 0}}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30 }, - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 }, }, { @@ -80282,7 +80282,7 @@ var OpcodeTable = [...]OpInfo{ {1, RegMask{V1: 33554432, V2: 0}}, // R26 {0, RegMask{V1: 1409286143, V2: 0}}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30 SP }, - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 }, }, { @@ -80295,7 +80295,7 @@ var OpcodeTable = [...]OpInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 335544319, V2: 0}}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 R30 }, - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 }, }, { @@ -80322,7 +80322,7 @@ var OpcodeTable = [...]OpInfo{ {1, RegMask{V1: 2, V2: 0}}, // R1 {2, RegMask{V1: 4, V2: 0}}, // R2 }, - Clobbers: RegMask{V1: 9223372035109945343, V2: 0}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071964721151, V2: 32767}, // R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15 R16 R17 R19 R20 R21 R22 R23 R24 R25 R26 g R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 Outputs: []OutputInfo{ {0, RegMask{V1: 1, V2: 0}}, // R0 }, @@ -81209,7 +81209,7 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 1, ClobberFlags: true, Reg: RegInfo{ - Clobbers: RegMask{V1: 9223372034975924224, V2: 0}, // R16 R17 R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + Clobbers: RegMask{V1: 18446744071830700032, V2: 32767}, // R16 R17 R30 F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 Outputs: []OutputInfo{ {0, RegMask{V1: 16777216, V2: 0}}, // R25 }, From cc71e4955cf1aea4299bfa3bc32b3c0e42327b31 Mon Sep 17 00:00:00 2001 From: Junyang Shao Date: Thu, 27 Aug 2026 15:29:10 -0400 Subject: [PATCH 3/5] simd/archsimd: add SVE Mul Lower to ZMUL for integers and ZFMUL for floats. The unpredicated integer ZMUL encoding is SVE2; its merging-predicated sibling and both float encodings are baseline SVE, so SVE2 buys only a better encoding, never availability. The loader records that split (cpuFeature is the operation's floor, unpredCpuFeature the carrier's level), the cpufeatures analysis now runs on arm64 and proves CPUsve2 from a dominating archsimd.ARM64.SVE2() check, and the generated rules use the unpredicated encoding only where that holds, falling back to the merging form under a synthesized all-true predicate: (MulInt8s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULB x y) (MulInt8s x y) => (ZMULMergingB x y ) A select over the fallback folds by replacing the all-true predicate, mirroring sveMergingPeephole's three forms. For #79781. Change-Id: I65c7b19b4686d6bb83ef7f9909ab2f42dcc910ec Reviewed-on: https://go-review.googlesource.com/c/go/+/822942 Reviewed-by: David Chase Auto-Submit: Junyang Shao Reviewed-by: Junyang Shao TryBot-Bypass: David Chase --- src/cmd/compile/internal/arm64/simdssa_sve.go | 18 + .../internal/ssa/_gen/simdARM64SVE.rules | 36 + .../internal/ssa/_gen/simdARM64SVEops.go | 18 + .../internal/ssa/_gen/simdgenericOps.go | 10 + src/cmd/compile/internal/ssa/block.go | 2 + src/cmd/compile/internal/ssa/ssaop/opGen.go | 376 ++++++++ .../internal/ssacompile/cpufeatures.go | 14 +- .../internal/ssagen/simdARM64SVEintrinsics.go | 10 + .../ssarewrite/rewritearm64/rewriteARM64.go | 860 +++++++++++++++++- .../archsimd/_gen/simdgen/gen_simdrules.go | 45 + .../archsimd/_gen/simdgen/ops/Mul/go_sve.yaml | 33 + src/simd/archsimd/_gen/simdgen/sve/emit.go | 17 +- .../archsimd/_gen/simdgen/sve/instruction.go | 5 + src/simd/archsimd/_gen/simdgen/sve/load.go | 4 +- .../archsimd/_gen/simdgen/types/operation.go | 7 + .../simd_test/binary_sve_arm64_test.go | 27 + src/simd/archsimd/ops_sve.go | 52 ++ test/codegen/simd_arm64.go | 13 +- 18 files changed, 1531 insertions(+), 16 deletions(-) create mode 100644 src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml diff --git a/src/cmd/compile/internal/arm64/simdssa_sve.go b/src/cmd/compile/internal/arm64/simdssa_sve.go index da1dff09f7f56d..d2ee3f9462b231 100644 --- a/src/cmd/compile/internal/arm64/simdssa_sve.go +++ b/src/cmd/compile/internal/arm64/simdssa_sve.go @@ -46,6 +46,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { case ssaop.OpARM64ZADDB, ssaop.OpARM64ZSQADDB, ssaop.OpARM64ZUQADDB, + ssaop.OpARM64ZMULB, ssaop.OpARM64ZSUBB, ssaop.OpARM64ZSQSUBB, ssaop.OpARM64ZUQSUBB: @@ -57,6 +58,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDD, ssaop.OpARM64ZANDD, ssaop.OpARM64ZBICD, + ssaop.OpARM64ZFMULD, + ssaop.OpARM64ZMULD, ssaop.OpARM64ZORRD, ssaop.OpARM64ZFSUBD, ssaop.OpARM64ZSUBD, @@ -68,6 +71,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { case ssaop.OpARM64ZADDH, ssaop.OpARM64ZSQADDH, ssaop.OpARM64ZUQADDH, + ssaop.OpARM64ZMULH, ssaop.OpARM64ZSUBH, ssaop.OpARM64ZSQSUBH, ssaop.OpARM64ZUQSUBH: @@ -77,6 +81,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZADDS, ssaop.OpARM64ZSQADDS, ssaop.OpARM64ZUQADDS, + ssaop.OpARM64ZFMULS, + ssaop.OpARM64ZMULS, ssaop.OpARM64ZFSUBS, ssaop.OpARM64ZSUBS, ssaop.OpARM64ZSQSUBS, @@ -128,6 +134,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingB, ssaop.OpARM64ZANDMergingB, ssaop.OpARM64ZBICMergingB, + ssaop.OpARM64ZMULMergingB, ssaop.OpARM64ZORRMergingB, ssaop.OpARM64ZSUBMergingB, ssaop.OpARM64ZSQSUBMergingB, @@ -141,6 +148,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingD, ssaop.OpARM64ZANDMergingD, ssaop.OpARM64ZBICMergingD, + ssaop.OpARM64ZFMULMergingD, + ssaop.OpARM64ZMULMergingD, ssaop.OpARM64ZORRMergingD, ssaop.OpARM64ZFSUBMergingD, ssaop.OpARM64ZSUBMergingD, @@ -154,6 +163,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingH, ssaop.OpARM64ZANDMergingH, ssaop.OpARM64ZBICMergingH, + ssaop.OpARM64ZMULMergingH, ssaop.OpARM64ZORRMergingH, ssaop.OpARM64ZSUBMergingH, ssaop.OpARM64ZSQSUBMergingH, @@ -167,6 +177,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingS, ssaop.OpARM64ZANDMergingS, ssaop.OpARM64ZBICMergingS, + ssaop.OpARM64ZFMULMergingS, + ssaop.OpARM64ZMULMergingS, ssaop.OpARM64ZORRMergingS, ssaop.OpARM64ZFSUBMergingS, ssaop.OpARM64ZSUBMergingS, @@ -201,6 +213,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDMergingPrefixedB, ssaop.OpARM64ZUQADDMergingPrefixedB, ssaop.OpARM64ZANDMergingPrefixedB, + ssaop.OpARM64ZMULMergingPrefixedB, ssaop.OpARM64ZORRMergingPrefixedB, ssaop.OpARM64ZEORMergingPrefixedB: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_B) @@ -210,6 +223,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDMergingPrefixedD, ssaop.OpARM64ZUQADDMergingPrefixedD, ssaop.OpARM64ZANDMergingPrefixedD, + ssaop.OpARM64ZFMULMergingPrefixedD, + ssaop.OpARM64ZMULMergingPrefixedD, ssaop.OpARM64ZORRMergingPrefixedD, ssaop.OpARM64ZEORMergingPrefixedD: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_D) @@ -218,6 +233,7 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDMergingPrefixedH, ssaop.OpARM64ZUQADDMergingPrefixedH, ssaop.OpARM64ZANDMergingPrefixedH, + ssaop.OpARM64ZMULMergingPrefixedH, ssaop.OpARM64ZORRMergingPrefixedH, ssaop.OpARM64ZEORMergingPrefixedH: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_H) @@ -227,6 +243,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDMergingPrefixedS, ssaop.OpARM64ZUQADDMergingPrefixedS, ssaop.OpARM64ZANDMergingPrefixedS, + ssaop.OpARM64ZFMULMergingPrefixedS, + ssaop.OpARM64ZMULMergingPrefixedS, ssaop.OpARM64ZORRMergingPrefixedS, ssaop.OpARM64ZEORMergingPrefixedS: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_S) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules index 75dfc96eab978d..48600f8e556728 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules @@ -34,6 +34,8 @@ (AndNotUint16s ...) => (ZBICD ...) // pureVreg (AndNotUint32s ...) => (ZBICD ...) // pureVreg (AndNotUint64s ...) => (ZBICD ...) // pureVreg +(MulFloat32s ...) => (ZFMULS ...) // pureVreg +(MulFloat64s ...) => (ZFMULD ...) // pureVreg (OrInt8s ...) => (ZORRD ...) // pureVreg (OrInt16s ...) => (ZORRD ...) // pureVreg (OrInt32s ...) => (ZORRD ...) // pureVreg @@ -108,6 +110,22 @@ (GreaterUint32s x y) => (ZCMPHIS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) (GreaterUint64s x y) => (ZCMPHID x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) (GreaterUint8s x y) => (ZCMPHIB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(MulInt16s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULH x y) +(MulInt16s x y) => (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) +(MulInt32s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULS x y) +(MulInt32s x y) => (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) +(MulInt64s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULD x y) +(MulInt64s x y) => (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) +(MulInt8s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULB x y) +(MulInt8s x y) => (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(MulUint16s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULH x y) +(MulUint16s x y) => (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) +(MulUint32s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULS x y) +(MulUint32s x y) => (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) +(MulUint64s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULD x y) +(MulUint64s x y) => (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) +(MulUint8s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULB x y) +(MulUint8s x y) => (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) (NegFloat32s x) => (ZFNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) (NegFloat64s x) => (ZFNEGD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) (NegInt16s x) => (ZNEGH x (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) @@ -141,6 +159,9 @@ (ZSELB (ZEORD x y) x mask) => (ZEORMergingB x y mask) (ZSELB (ZEORD x y) y mask) => (ZEORMergingB y x mask) (ZSELB (ZEORD x y) z mask) => (ZEORMergingPrefixedB z x y mask) +(ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) => (ZMULMergingB x y mask) +(ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) => (ZMULMergingB y x mask) +(ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) => (ZMULMergingPrefixedB z x y mask) (ZSELB (ZNEGB x (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) => (ZNEGMergingB z x mask) (ZSELB (ZORRD x y) x mask) => (ZORRMergingB x y mask) (ZSELB (ZORRD x y) y mask) => (ZORRMergingB y x mask) @@ -169,9 +190,15 @@ (ZSELD (ZFADDD x y) x mask) => (ZFADDMergingD x y mask) (ZSELD (ZFADDD x y) y mask) => (ZFADDMergingD y x mask) (ZSELD (ZFADDD x y) z mask) => (ZFADDMergingPrefixedD z x y mask) +(ZSELD (ZFMULD x y) x mask) => (ZFMULMergingD x y mask) +(ZSELD (ZFMULD x y) y mask) => (ZFMULMergingD y x mask) +(ZSELD (ZFMULD x y) z mask) => (ZFMULMergingPrefixedD z x y mask) (ZSELD (ZFNEGD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZFNEGMergingD z x mask) (ZSELD (ZFSQRTD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZFSQRTMergingD z x mask) (ZSELD (ZFSUBD x y) x mask) => (ZFSUBMergingD x y mask) +(ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) => (ZMULMergingD x y mask) +(ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) => (ZMULMergingD y x mask) +(ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZMULMergingPrefixedD z x y mask) (ZSELD (ZNEGD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZNEGMergingD z x mask) (ZSELD (ZORRD x y) x mask) => (ZORRMergingD x y mask) (ZSELD (ZORRD x y) y mask) => (ZORRMergingD y x mask) @@ -196,6 +223,9 @@ (ZSELH (ZEORD x y) x mask) => (ZEORMergingH x y mask) (ZSELH (ZEORD x y) y mask) => (ZEORMergingH y x mask) (ZSELH (ZEORD x y) z mask) => (ZEORMergingPrefixedH z x y mask) +(ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) => (ZMULMergingH x y mask) +(ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) => (ZMULMergingH y x mask) +(ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) => (ZMULMergingPrefixedH z x y mask) (ZSELH (ZNEGH x (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) => (ZNEGMergingH z x mask) (ZSELH (ZORRD x y) x mask) => (ZORRMergingH x y mask) (ZSELH (ZORRD x y) y mask) => (ZORRMergingH y x mask) @@ -224,9 +254,15 @@ (ZSELS (ZFADDS x y) x mask) => (ZFADDMergingS x y mask) (ZSELS (ZFADDS x y) y mask) => (ZFADDMergingS y x mask) (ZSELS (ZFADDS x y) z mask) => (ZFADDMergingPrefixedS z x y mask) +(ZSELS (ZFMULS x y) x mask) => (ZFMULMergingS x y mask) +(ZSELS (ZFMULS x y) y mask) => (ZFMULMergingS y x mask) +(ZSELS (ZFMULS x y) z mask) => (ZFMULMergingPrefixedS z x y mask) (ZSELS (ZFNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZFNEGMergingS z x mask) (ZSELS (ZFSQRTS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZFSQRTMergingS z x mask) (ZSELS (ZFSUBS x y) x mask) => (ZFSUBMergingS x y mask) +(ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) => (ZMULMergingS x y mask) +(ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) => (ZMULMergingS y x mask) +(ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZMULMergingPrefixedS z x y mask) (ZSELS (ZNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZNEGMergingS z x mask) (ZSELS (ZORRD x y) x mask) => (ZORRMergingS x y mask) (ZSELS (ZORRD x y) y mask) => (ZORRMergingS y x mask) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go index 87dd20e3ce44df..062387c5a18150 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go @@ -89,6 +89,12 @@ func simdARM64SVEOps(z11, z21, zkv, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []op {name: "ZFCMGTS", argLength: 3, reg: z2kk, asm: "ZFCMGT", typ: "Mask"}, {name: "ZFCMNED", argLength: 3, reg: z2kk, asm: "ZFCMNE", commutative: true, typ: "Mask"}, {name: "ZFCMNES", argLength: 3, reg: z2kk, asm: "ZFCMNE", commutative: true, typ: "Mask"}, + {name: "ZFMULD", argLength: 2, reg: z21, asm: "ZFMUL", commutative: true, typ: "Vec256"}, + {name: "ZFMULMergingD", argLength: 3, reg: z2kvPred, asm: "ZFMUL", commutative: true, typ: "Vec256"}, + {name: "ZFMULMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZFMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZFMULMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZFMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZFMULMergingS", argLength: 3, reg: z2kvPred, asm: "ZFMUL", commutative: true, typ: "Vec256"}, + {name: "ZFMULS", argLength: 2, reg: z21, asm: "ZFMUL", commutative: true, typ: "Vec256"}, {name: "ZFNEGD", argLength: 2, reg: zkv, asm: "ZFNEG", typ: "Vec256"}, {name: "ZFNEGMergingD", argLength: 3, reg: z2kvPred, asm: "ZFNEG", typ: "Vec256", resultInArg0: true}, {name: "ZFNEGMergingS", argLength: 3, reg: z2kvPred, asm: "ZFNEG", typ: "Vec256", resultInArg0: true}, @@ -109,6 +115,18 @@ func simdARM64SVEOps(z11, z21, zkv, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []op {name: "ZFSUBMergingD", argLength: 3, reg: z2kvPred, asm: "ZFSUB", typ: "Vec256", resultInArg0: true}, {name: "ZFSUBMergingS", argLength: 3, reg: z2kvPred, asm: "ZFSUB", typ: "Vec256", resultInArg0: true}, {name: "ZFSUBS", argLength: 2, reg: z21, asm: "ZFSUB", typ: "Vec256"}, + {name: "ZMULB", argLength: 2, reg: z21, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULD", argLength: 2, reg: z21, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULH", argLength: 2, reg: z21, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULMergingB", argLength: 3, reg: z2kvPred, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULMergingD", argLength: 3, reg: z2kvPred, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULMergingH", argLength: 3, reg: z2kvPred, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZMULMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZMULMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZMULMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZMUL", typ: "Vec256", resultInArg0: true}, + {name: "ZMULMergingS", argLength: 3, reg: z2kvPred, asm: "ZMUL", commutative: true, typ: "Vec256"}, + {name: "ZMULS", argLength: 2, reg: z21, asm: "ZMUL", commutative: true, typ: "Vec256"}, {name: "ZNEGB", argLength: 2, reg: zkv, asm: "ZNEG", typ: "Vec256"}, {name: "ZNEGD", argLength: 2, reg: zkv, asm: "ZNEG", typ: "Vec256"}, {name: "ZNEGH", argLength: 2, reg: zkv, asm: "ZNEG", typ: "Vec256"}, diff --git a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go index 81d205033c28d8..507b37a8663408 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go @@ -818,9 +818,11 @@ func simdGenericOps() []opData { {name: "MulAddUint8x16", argLength: 3}, // ARCH:arm64 {name: "MulAddUint16x8", argLength: 3}, // ARCH:arm64 {name: "MulAddUint32x4", argLength: 3}, // ARCH:arm64 + {name: "MulFloat32s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulFloat32x4", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulFloat32x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulFloat32x16", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulFloat64s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulFloat64x2", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulFloat64x4", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulFloat64x8", argLength: 2, commutative: true}, // ARCH:amd64 @@ -830,13 +832,17 @@ func simdGenericOps() []opData { {name: "MulHighUint16x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighUint16x16", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighUint16x32", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulInt8s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt8x16", argLength: 2, commutative: true}, // ARCH:arm64 + {name: "MulInt16s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt16x8", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulInt16x16", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulInt16x32", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulInt32s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt32x4", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulInt32x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulInt32x16", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulInt64s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt64x2", argLength: 2, commutative: true}, // ARCH:amd64,wasm {name: "MulInt64x4", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulInt64x8", argLength: 2, commutative: true}, // ARCH:amd64 @@ -846,13 +852,17 @@ func simdGenericOps() []opData { {name: "MulSignInt16x16", argLength: 2}, // ARCH:amd64 {name: "MulSignInt32x4", argLength: 2}, // ARCH:amd64 {name: "MulSignInt32x8", argLength: 2}, // ARCH:amd64 + {name: "MulUint8s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulUint8x16", argLength: 2, commutative: true}, // ARCH:arm64 + {name: "MulUint16s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulUint16x8", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulUint16x16", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulUint16x32", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulUint32s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulUint32x4", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulUint32x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulUint32x16", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulUint64s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulUint64x2", argLength: 2, commutative: true}, // ARCH:amd64,wasm {name: "MulUint64x4", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulUint64x8", argLength: 2, commutative: true}, // ARCH:amd64 diff --git a/src/cmd/compile/internal/ssa/block.go b/src/cmd/compile/internal/ssa/block.go index 5981fee5fbe13b..522e62e64d1cd4 100644 --- a/src/cmd/compile/internal/ssa/block.go +++ b/src/cmd/compile/internal/ssa/block.go @@ -507,6 +507,8 @@ func (f CPUfeatures) String() string { foo("vbmi2", CPUvbmi2) foo("popcntdq", CPUvpopcntdq) foo("avx512vnni", CPUavx512vnni) + foo("neon", CPUneon) + foo("sve2", CPUsve2) return s } diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index cc2582229d5f55..1701259349eb42 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -5125,6 +5125,12 @@ const ( OpARM64ZFCMGTS OpARM64ZFCMNED OpARM64ZFCMNES + OpARM64ZFMULD + OpARM64ZFMULMergingD + OpARM64ZFMULMergingPrefixedD + OpARM64ZFMULMergingPrefixedS + OpARM64ZFMULMergingS + OpARM64ZFMULS OpARM64ZFNEGD OpARM64ZFNEGMergingD OpARM64ZFNEGMergingS @@ -5145,6 +5151,18 @@ const ( OpARM64ZFSUBMergingD OpARM64ZFSUBMergingS OpARM64ZFSUBS + OpARM64ZMULB + OpARM64ZMULD + OpARM64ZMULH + OpARM64ZMULMergingB + OpARM64ZMULMergingD + OpARM64ZMULMergingH + OpARM64ZMULMergingPrefixedB + OpARM64ZMULMergingPrefixedD + OpARM64ZMULMergingPrefixedH + OpARM64ZMULMergingPrefixedS + OpARM64ZMULMergingS + OpARM64ZMULS OpARM64ZNEGB OpARM64ZNEGD OpARM64ZNEGH @@ -7996,9 +8014,11 @@ const ( OpMulAddUint16x8 OpMulAddUint32x4 OpMulAddUint8x16 + OpMulFloat32s OpMulFloat32x16 OpMulFloat32x4 OpMulFloat32x8 + OpMulFloat64s OpMulFloat64x2 OpMulFloat64x4 OpMulFloat64x8 @@ -8008,15 +8028,19 @@ const ( OpMulHighUint16x16 OpMulHighUint16x32 OpMulHighUint16x8 + OpMulInt16s OpMulInt16x16 OpMulInt16x32 OpMulInt16x8 + OpMulInt32s OpMulInt32x16 OpMulInt32x4 OpMulInt32x8 + OpMulInt64s OpMulInt64x2 OpMulInt64x4 OpMulInt64x8 + OpMulInt8s OpMulInt8x16 OpMulSignInt16x16 OpMulSignInt16x8 @@ -8024,15 +8048,19 @@ const ( OpMulSignInt32x8 OpMulSignInt8x16 OpMulSignInt8x32 + OpMulUint16s OpMulUint16x16 OpMulUint16x32 OpMulUint16x8 + OpMulUint32s OpMulUint32x16 OpMulUint32x4 OpMulUint32x8 + OpMulUint64s OpMulUint64x2 OpMulUint64x4 OpMulUint64x8 + OpMulUint8s OpMulUint8x16 OpMulWidenEvenInt32x4 OpMulWidenEvenInt32x8 @@ -87242,6 +87270,102 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZFMULD", + ArgLen: 2, + Commutative: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFMULMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFMULMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFMULMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFMULMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZFMULS", + ArgLen: 2, + Commutative: true, + asm: arm64.AZFMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZFNEGD", ArgLen: 2, @@ -87534,6 +87658,198 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZMULB", + ArgLen: 2, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULD", + ArgLen: 2, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULH", + ArgLen: 2, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZMULS", + ArgLen: 2, + Commutative: true, + asm: arm64.AZMUL, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZNEGB", ArgLen: 2, @@ -116624,6 +116940,12 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 3, Generic: true, }, + { + Name: "MulFloat32s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulFloat32x16", ArgLen: 2, @@ -116642,6 +116964,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulFloat64s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulFloat64x2", ArgLen: 2, @@ -116696,6 +117024,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulInt16s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulInt16x16", ArgLen: 2, @@ -116714,6 +117048,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulInt32s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulInt32x16", ArgLen: 2, @@ -116732,6 +117072,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulInt64s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulInt64x2", ArgLen: 2, @@ -116750,6 +117096,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulInt8s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulInt8x16", ArgLen: 2, @@ -116786,6 +117138,12 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 2, Generic: true, }, + { + Name: "MulUint16s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulUint16x16", ArgLen: 2, @@ -116804,6 +117162,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulUint32s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulUint32x16", ArgLen: 2, @@ -116822,6 +117186,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulUint64s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulUint64x2", ArgLen: 2, @@ -116840,6 +117210,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulUint8s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulUint8x16", ArgLen: 2, diff --git a/src/cmd/compile/internal/ssacompile/cpufeatures.go b/src/cmd/compile/internal/ssacompile/cpufeatures.go index cccac88e6063bd..9bdb0d9f16b8b3 100644 --- a/src/cmd/compile/internal/ssacompile/cpufeatures.go +++ b/src/cmd/compile/internal/ssacompile/cpufeatures.go @@ -52,7 +52,7 @@ func ifEffect(b *ssa.Block) (features ssa.CPUfeatures, taken int) { return } sym := addr.Aux.(*obj.LSym) - if sym.Name != "internal/cpu.X86" { + if sym.Name != "internal/cpu.X86" && sym.Name != "internal/cpu.ARM64" { return } o := offPtr.AuxInt @@ -104,7 +104,12 @@ func ifEffect(b *ssa.Block) (features ssa.CPUfeatures, taken int) { case "HasBMI2": features = ssa.CPUvbmi2 + case "HasSVE2": + features = ssa.CPUsve2 + // Features that are not currently interesting to the compiler. + // HasSVE is not among them: rules only ever upgrade an SVE lowering to an + // SVE2 encoding, so baseline SVE is never a rule condition. case "HasAES", "HasADX", "HasERMS", "HasFSRM", "HasFMA", "HasGFNI", "HasOSXSAVE", "HasPCLMULQDQ", "HasPOPCNT", "HasRDTSCP", "HasSHA", "HasSSE3", "HasSSSE3", "HasSSE41", "HasSSE42": @@ -130,7 +135,7 @@ func noCodeValue(v *ssa.Value) bool { func cpufeatures(f *ssa.Func) { arch := f.Config.Ctxt.Arch.Family // TODO there are other SIMD architectures - if arch != goarch.AMD64 { + if arch != goarch.AMD64 && arch != goarch.ARM64 { return } @@ -139,7 +144,10 @@ func cpufeatures(f *ssa.Func) { effects := make([]localEffect, 1+f.NumBlocks(), 1+f.NumBlocks()) features := func(t *types.Type) ssa.CPUfeatures { - if t.IsSIMD() { + if t.IsSIMD() && arch == goarch.AMD64 { + // On arm64 a SIMD type implies no feature a rule conditions on: + // rules only upgrade SVE lowerings to SVE2 encodings, and using an + // SVE type does not imply SVE2. switch t.Size() { case 16, 32: return ssa.CPUavx diff --git a/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go b/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go index 06b0c3a0645226..f9111340306048 100644 --- a/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go +++ b/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go @@ -85,6 +85,16 @@ func simdARM64SVEIntrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFa addF(simdPackage, "Uint16s.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint16s, types.TypeMask), sys.ARM64) addF(simdPackage, "Uint32s.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint32s, types.TypeMask), sys.ARM64) addF(simdPackage, "Uint64s.GreaterEqual", opLen2(ssaop.OpGreaterEqualUint64s, types.TypeMask), sys.ARM64) + addF(simdPackage, "Float32s.Mul", opLen2(ssaop.OpMulFloat32s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Float64s.Mul", opLen2(ssaop.OpMulFloat64s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int8s.Mul", opLen2(ssaop.OpMulInt8s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int16s.Mul", opLen2(ssaop.OpMulInt16s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int32s.Mul", opLen2(ssaop.OpMulInt32s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int64s.Mul", opLen2(ssaop.OpMulInt64s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint8s.Mul", opLen2(ssaop.OpMulUint8s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint16s.Mul", opLen2(ssaop.OpMulUint16s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint32s.Mul", opLen2(ssaop.OpMulUint32s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint64s.Mul", opLen2(ssaop.OpMulUint64s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Float32s.Neg", opLen1(ssaop.OpNegFloat32s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Float64s.Neg", opLen1(ssaop.OpNegFloat64s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Int8s.Neg", opLen1(ssaop.OpNegInt8s, types.TypeVec256), sys.ARM64) diff --git a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go index f56d229d482fe6..a3d8ea9de1035c 100644 --- a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go +++ b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go @@ -1841,27 +1841,49 @@ func RewriteValue(v *ssa.Value) bool { return rewriteValue_OpMulAddUint32x4(v) case ssaop.OpMulAddUint8x16: return rewriteValue_OpMulAddUint8x16(v) + case ssaop.OpMulFloat32s: + v.Op = ssaop.OpARM64ZFMULS + return true case ssaop.OpMulFloat32x4: v.Op = ssaop.OpARM64VFMUL4S return true + case ssaop.OpMulFloat64s: + v.Op = ssaop.OpARM64ZFMULD + return true case ssaop.OpMulFloat64x2: v.Op = ssaop.OpARM64VFMUL2D return true + case ssaop.OpMulInt16s: + return rewriteValue_OpMulInt16s(v) case ssaop.OpMulInt16x8: v.Op = ssaop.OpARM64VMUL8H return true + case ssaop.OpMulInt32s: + return rewriteValue_OpMulInt32s(v) case ssaop.OpMulInt32x4: v.Op = ssaop.OpARM64VMUL4S return true + case ssaop.OpMulInt64s: + return rewriteValue_OpMulInt64s(v) + case ssaop.OpMulInt8s: + return rewriteValue_OpMulInt8s(v) case ssaop.OpMulInt8x16: v.Op = ssaop.OpARM64VMUL16B return true + case ssaop.OpMulUint16s: + return rewriteValue_OpMulUint16s(v) case ssaop.OpMulUint16x8: v.Op = ssaop.OpARM64VMUL8H return true + case ssaop.OpMulUint32s: + return rewriteValue_OpMulUint32s(v) case ssaop.OpMulUint32x4: v.Op = ssaop.OpARM64VMUL4S return true + case ssaop.OpMulUint64s: + return rewriteValue_OpMulUint64s(v) + case ssaop.OpMulUint8s: + return rewriteValue_OpMulUint8s(v) case ssaop.OpMulUint8x16: v.Op = ssaop.OpARM64VMUL16B return true @@ -20331,6 +20353,110 @@ func rewriteValue_OpARM64ZSELB(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) + // result: (ZMULMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) + // result: (ZMULMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) + // result: (ZMULMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingB { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELB (ZNEGB x (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) // result: (ZNEGMergingB z x mask) for { @@ -20900,6 +21026,64 @@ func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELD (ZFMULD x y) x mask) + // result: (ZFMULMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFMULD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZFMULD x y) y mask) + // result: (ZFMULMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFMULD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZFMULD x y) z mask) + // result: (ZFMULMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFMULD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELD (ZFNEGD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) // result: (ZFNEGMergingD z x mask) for { @@ -20978,6 +21162,110 @@ func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { v.AddArg3(x, y, mask) return true } + // match: (ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) + // result: (ZMULMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingD { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) + // result: (ZMULMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingD { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) + // result: (ZMULMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingD { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELD (ZNEGD x (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) // result: (ZNEGMergingD z x mask) for { @@ -21458,6 +21746,110 @@ func rewriteValue_OpARM64ZSELH(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) + // result: (ZMULMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) + // result: (ZMULMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) + // result: (ZMULMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingH { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELH (ZNEGH x (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) // result: (ZNEGMergingH z x mask) for { @@ -22027,23 +22419,81 @@ func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } - // match: (ZSELS (ZFNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) - // result: (ZFNEGMergingS z x mask) + // match: (ZSELS (ZFMULS x y) x mask) + // result: (ZFMULMergingS x y mask) for { - if v_0.Op != ssaop.OpARM64ZFNEGS { + if v_0.Op != ssaop.OpARM64ZFMULS { break } _ = v_0.Args[1] - x := v_0.Args[0] + v_0_0 := v_0.Args[0] v_0_1 := v_0.Args[1] - if v_0_1.Op != ssaop.OpSelect0 || v_0_1.Type != types.TypeMask { - break + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingS) + v.AddArg3(x, y, mask) + return true } - v_0_1_0 := v_0_1.Args[0] - if v_0_1_0.Op != ssaop.OpARM64PWHILELTS { + break + } + // match: (ZSELS (ZFMULS x y) y mask) + // result: (ZFMULMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZFMULS { break } - _ = v_0_1_0.Args[1] + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZFMULS x y) z mask) + // result: (ZFMULMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZFMULS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZFMULMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELS (ZFNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) + // result: (ZFNEGMergingS z x mask) + for { + if v_0.Op != ssaop.OpARM64ZFNEGS { + break + } + _ = v_0.Args[1] + x := v_0.Args[0] + v_0_1 := v_0.Args[1] + if v_0_1.Op != ssaop.OpSelect0 || v_0_1.Type != types.TypeMask { + break + } + v_0_1_0 := v_0_1.Args[0] + if v_0_1_0.Op != ssaop.OpARM64PWHILELTS { + break + } + _ = v_0_1_0.Args[1] v_0_1_0_0 := v_0_1_0.Args[0] if v_0_1_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_1_0_0.AuxInt) != 0 { break @@ -22105,6 +22555,110 @@ func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { v.AddArg3(x, y, mask) return true } + // match: (ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) + // result: (ZMULMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) + // result: (ZMULMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) + // result: (ZMULMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZMULMergingS { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZMULMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELS (ZNEGS x (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) // result: (ZNEGMergingS z x mask) for { @@ -25875,6 +26429,294 @@ func rewriteValue_OpMulAddUint8x16(v *ssa.Value) bool { return true } } +func rewriteValue_OpMulInt16s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulInt16s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULH x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULH) + v.AddArg2(x, y) + return true + } + // match: (MulInt16s x y) + // result: (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingH) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTH, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(16) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulInt32s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulInt32s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULS x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULS) + v.AddArg2(x, y) + return true + } + // match: (MulInt32s x y) + // result: (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingS) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTS, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(8) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulInt64s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulInt64s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULD x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULD) + v.AddArg2(x, y) + return true + } + // match: (MulInt64s x y) + // result: (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingD) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTD, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(4) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulInt8s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulInt8s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULB x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULB) + v.AddArg2(x, y) + return true + } + // match: (MulInt8s x y) + // result: (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingB) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTB, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(32) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulUint16s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulUint16s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULH x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULH) + v.AddArg2(x, y) + return true + } + // match: (MulUint16s x y) + // result: (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingH) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTH, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(16) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulUint32s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulUint32s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULS x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULS) + v.AddArg2(x, y) + return true + } + // match: (MulUint32s x y) + // result: (ZMULMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingS) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTS, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(8) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulUint64s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulUint64s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULD x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULD) + v.AddArg2(x, y) + return true + } + // match: (MulUint64s x y) + // result: (ZMULMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingD) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTD, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(4) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulUint8s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulUint8s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZMULB x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZMULB) + v.AddArg2(x, y) + return true + } + // match: (MulUint8s x y) + // result: (ZMULMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZMULMergingB) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTB, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(32) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} func rewriteValue_OpNegFloat32s(v *ssa.Value) bool { v_0 := v.Args[0] b := v.Block diff --git a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go index 1eebd1d2afd7dc..c4dda3865c5a33 100644 --- a/src/simd/archsimd/_gen/simdgen/gen_simdrules.go +++ b/src/simd/archsimd/_gen/simdgen/gen_simdrules.go @@ -214,6 +214,16 @@ func sveImplicitPredRule(gOp Operation, asm, args string) string { gOp.GenericName(), args, asm, args, letter, lanes) } +// sveFeatureConst maps a def's cpuFeature string to the ssa.CPUfeatures +// constant the generated rule conditions on. +func sveFeatureConst(feature string) string { + switch feature { + case "SVE2": + return "CPUsve2" + } + panic(fmt.Errorf("simdgen: no ssa.CPUfeatures constant for feature %q", feature)) +} + // sveAllTruePattern returns the rule text for the synthesized all-true // governing predicate of an operation, the same term sveImplicitPredRule // produces. Matching it, rather than a wildcard, is what makes the peepholes @@ -356,6 +366,41 @@ func writeSIMDRules(buffer *bytes.Buffer, ops []Operation) { data.ArgsOut = fmt.Sprintf("[a+%s] %s", *opr.In[0].Const, data.ArgsOut) } + // An SVE operation whose unpredicated encoding needs a higher feature + // level than the operation itself lowers through its merging-predicated + // sibling under a synthesized all-true predicate, with the unpredicated + // encoding as a feature-gated upgrade in blocks where the cpufeatures + // analysis proves the level: + // + // (MulInt8s x y) && ...HasFeature(ssa.CPUsve2) => (ZMULB x y) + // (MulInt8s x y) => (ZMULMergingB x y ) + if gOp.UnpredCPUFeature != nil { + preds := gOp.svePredicatedOps() + if len(preds) == 0 || sveMaskSuffix(preds[0]) != "Merging" { + panic(fmt.Errorf("simdgen: %s has unpredCpuFeature but no merging fallback", gOp.Asm)) + } + merging := machineOpName(OneMask, preds[0]) + sveRules = append(sveRules, fmt.Sprintf("(%s %s) && v.Block.CPUfeatures.HasFeature(ssa.%s) => (%s %s)\n", + gOp.GenericName(), data.Args, sveFeatureConst(*gOp.UnpredCPUFeature), asm, data.Args)) + sveRules = append(sveRules, fmt.Sprintf("(%s %s) => (%s %s %s)\n", + gOp.GenericName(), data.Args, merging, data.Args, sveAllTruePattern(gOp))) + // A select over the fallback folds by replacing the synthesized + // all-true predicate, mirroring sveMergingPeephole's three forms. + sel := "ZSEL" + sveArrangementLetter(gOp) + allTrue := sveAllTruePattern(gOp) + sveRules = append(sveRules, fmt.Sprintf("(%s (%s x y %s) x mask) => (%s x y mask)\n", sel, merging, allTrue, merging)) + if gOp.Commutative { + sveRules = append(sveRules, fmt.Sprintf("(%s (%s x y %s) y mask) => (%s y x mask)\n", sel, merging, allTrue, merging)) + if preds[0].sveMergingPrefixedOp() != nil { + sveRules = append(sveRules, fmt.Sprintf("(%s (%s x y %s) z mask) => (%s z x y mask)\n", + sel, merging, allTrue, machineOpName(OneMask, *preds[0].sveMergingPrefixedOp()))) + } + } + asmCheck[asm] = true + asmCheck[merging] = true + continue + } + // SVE ops with an implicit-all-true governing predicate expose an // unpredicated Go API: lower the generic op straight to the predicated // machine op, synthesizing an all-true predicate. This bypasses the AVX diff --git a/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml b/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml new file mode 100644 index 00000000000000..a086bd129af86b --- /dev/null +++ b/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml @@ -0,0 +1,33 @@ +!sum +# Mul — integer multiply (low half), ZMUL. The unpredicated vector encoding is +# SVE2; the merging-predicated sibling is baseline SVE. +- go: Mul + asm: "ZMUL" + in: + - &int + go: $t + base: int + - *int + out: + - *int + +- go: Mul + asm: "ZMUL" + in: + - &uint + go: $t + base: uint + - *uint + out: + - *uint + +# Mul — floating-point multiply, ZFMUL (baseline SVE, both forms). +- go: Mul + asm: "ZFMUL" + in: + - &float + go: $t + base: float + - *float + out: + - *float diff --git a/src/simd/archsimd/_gen/simdgen/sve/emit.go b/src/simd/archsimd/_gen/simdgen/sve/emit.go index 8e4887e1a51d7b..690c869cbf665e 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/emit.go +++ b/src/simd/archsimd/_gen/simdgen/sve/emit.go @@ -131,7 +131,22 @@ func (inst *Instruction) emitOne(asm string, ops []Operand, widthAgnostic bool) var db unify.DefBuilder db.Add("asm", unify.NewValue(unify.NewStringExact(asm))) db.Add("goarch", unify.NewValue(unify.NewStringExact("arm64"))) - db.Add("cpuFeature", unify.NewValue(unify.NewStringExact(inst.cpuFeature()))) + // The operation's feature level is the floor across its encodings: an + // operation whose predicated sibling is baseline SVE is available on SVE + // even when its unpredicated carrier needs SVE2 — the carrier is then a + // feature-gated upgrade, recorded as unpredCpuFeature for the rules. + feature := inst.cpuFeature() + unpred := "" + for _, pv := range inst.predVariants { + if pv.cpuFeature == "SVE" && feature == "SVE2" { + unpred = feature + feature = pv.cpuFeature + } + } + db.Add("cpuFeature", unify.NewValue(unify.NewStringExact(feature))) + if unpred != "" { + db.Add("unpredCPUFeature", unify.NewValue(unify.NewStringExact(unpred))) + } if doc := inst.documentation(); doc != "" { db.Add("details", unify.NewValue(unify.NewStringExact(asComment(doc, 80)))) } diff --git a/src/simd/archsimd/_gen/simdgen/sve/instruction.go b/src/simd/archsimd/_gen/simdgen/sve/instruction.go index 310bbc0fc78c54..1bffcfe979cafc 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/instruction.go +++ b/src/simd/archsimd/_gen/simdgen/sve/instruction.go @@ -87,6 +87,11 @@ type predVariant struct { quals string outRegNames []string inRegNames []string + // cpuFeature is the predicated sibling encoding's own feature level. It can + // sit below the carrier's: the unpredicated integer MUL is SVE2 while its + // predicated sibling is baseline SVE, making the operation available on SVE + // with the unpredicated encoding a feature-gated upgrade. + cpuFeature string // predAsmPos is the assembly position of the encoding's governing // predicate: 1 on every encoding grouped today, but recorded rather than // assumed — PTEST, with no destination, governs from position 0. diff --git a/src/simd/archsimd/_gen/simdgen/sve/load.go b/src/simd/archsimd/_gen/simdgen/sve/load.go index 9b16deaa3be321..4df452ce5c9db9 100644 --- a/src/simd/archsimd/_gen/simdgen/sve/load.go +++ b/src/simd/archsimd/_gen/simdgen/sve/load.go @@ -128,7 +128,7 @@ func groupPredicationForms(insts []*Instruction) map[*Instruction]bool { // Each encoding carries its own register symbols, so a machine op is // always generated from the shape of the encoding it comes from. outs, ins := splitRegNames(prOps) - variants = append(variants, predVariant{quals: quals, outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(prOps)}) + variants = append(variants, predVariant{quals: quals, outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(prOps), cpuFeature: pr.cpuFeature()}) covered[pr] = true } if len(variants) == 0 { @@ -185,7 +185,7 @@ func groupPredicatedOnly(pred []*Instruction, covered map[*Instruction]bool) { } baseOps := base.operands() outs, ins := splitRegNames(baseOps) - base.predVariants = []predVariant{{quals: "M", outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(baseOps)}} + base.predVariants = []predVariant{{quals: "M", outRegNames: outs, inRegNames: ins, predAsmPos: governingAsmPos(baseOps), cpuFeature: base.cpuFeature()}} for _, inst := range byQual { if inst != base { covered[inst] = true diff --git a/src/simd/archsimd/_gen/simdgen/types/operation.go b/src/simd/archsimd/_gen/simdgen/types/operation.go index 9bfccd3e3e5a05..9fc619420c7048 100644 --- a/src/simd/archsimd/_gen/simdgen/types/operation.go +++ b/src/simd/archsimd/_gen/simdgen/types/operation.go @@ -51,6 +51,13 @@ type RawOperation struct { // If true, do not emit method declarations, generic ops, or intrinsics for masked variants // DO emit the architecture-specific opcodes and optimizations. HideMaskMethods *bool + // UnpredCPUFeature is set when an SVE operation's unpredicated encoding + // needs a higher feature level than the operation itself: the CPUFeature + // field is the floor its predicated sibling provides, and the lowering may + // use the unpredicated encoding only in blocks where the ssa cpufeatures + // analysis proves this level (e.g. integer MUL: CPUFeature "SVE", + // UnpredCPUFeature "SVE2"). + UnpredCPUFeature *string // WidthAgnostic marks an SVE bitwise operation whose unpredicated encoding // is written .D but computes the same bits under any element view. The // loader emits one def per element width so every Go type gets the API; diff --git a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go index d4e3cf14a8a406..cbfee86b87129d 100644 --- a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go @@ -335,3 +335,30 @@ func TestAndNotSVE(t *testing.T) { } testInt8sBinary(t, archsimd.Int8s.AndNot, andNotInt8) } + +func TestMulSVE(t *testing.T) { + if !archsimd.ARM64.SVE() { + t.Skip("no SVE") + } + mulFloat64 := func(x, y []float64) []float64 { + r := make([]float64, len(x)) + for i := range x { + r[i] = x[i] * y[i] + } + return r + } + testFloat64sBinary(t, archsimd.Float64s.Mul, mulFloat64) + mulInt8 := func(x, y []int8) []int8 { + r := make([]int8, len(x)) + for i := range x { + r[i] = x[i] * y[i] + } + return r + } + // Ungated: compiles to the merging-predicated fallback, correct on any SVE. + testInt8sBinary(t, archsimd.Int8s.Mul, mulInt8) + if archsimd.ARM64.SVE2() { + // Gated: this block compiles to the unpredicated SVE2 encoding. + testInt8sBinary(t, archsimd.Int8s.Mul, mulInt8) + } +} diff --git a/src/simd/archsimd/ops_sve.go b/src/simd/archsimd/ops_sve.go index 9004f6ac2b2f97..d3e70668f2b334 100644 --- a/src/simd/archsimd/ops_sve.go +++ b/src/simd/archsimd/ops_sve.go @@ -394,6 +394,58 @@ func (x Uint32s) GreaterEqual(y Uint32s) Mask32s // Asm: ZCMPHS, CPU Feature: SVE func (x Uint64s) GreaterEqual(y Uint64s) Mask64s +/* Mul */ + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZFMUL, CPU Feature: SVE +func (x Float32s) Mul(y Float32s) Float32s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZFMUL, CPU Feature: SVE +func (x Float64s) Mul(y Float64s) Float64s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Int8s) Mul(y Int8s) Int8s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Int16s) Mul(y Int16s) Int16s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Int32s) Mul(y Int32s) Int32s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Int64s) Mul(y Int64s) Int64s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Uint8s) Mul(y Uint8s) Uint8s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Uint16s) Mul(y Uint16s) Uint16s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Uint32s) Mul(y Uint32s) Uint32s + +// Mul multiplies corresponding elements of two vectors, modulo 2ⁿ. +// +// Asm: ZMUL, CPU Feature: SVE +func (x Uint64s) Mul(y Uint64s) Uint64s + /* Neg */ // Neg returns the elementwise negation of x. diff --git a/test/codegen/simd_arm64.go b/test/codegen/simd_arm64.go index 98edf4cc0395af..3ba251f9127727 100644 --- a/test/codegen/simd_arm64.go +++ b/test/codegen/simd_arm64.go @@ -255,4 +255,15 @@ func sveIfElseFloat(x, y archsimd.Float64s, m archsimd.Mask64s) archsimd.Float64 func sveZeroMask() archsimd.Mask8s { // arm64:`PPFALSE` -`ZDUP` var m archsimd.Mask8s - return m} + return m +} + +// An operation whose unpredicated encoding needs SVE2 lowers to it only in +// blocks where the cpufeatures analysis proves SVE2; elsewhere it lowers to +// its baseline-SVE merging-predicated sibling under an all-true predicate. +func sveMulSVE2Gate(x, y archsimd.Int8s) archsimd.Int8s { + if archsimd.ARM64.SVE2() { + return x.Mul(y) // arm64:`ZMUL\s+Z[0-9]+\.B, Z[0-9]+\.B, Z[0-9]+\.B` + } + return x.Mul(y) // arm64:`PWHILELT` `ZMUL.*P[0-9]+\.M` +} From 76b21f0229db1156c7117a8e80879ce29474c052 Mon Sep 17 00:00:00 2001 From: Junyang Shao Date: Thu, 27 Aug 2026 15:30:15 -0400 Subject: [PATCH 4/5] simd/archsimd: add SVE MulHigh Lower to ZSMULH and ZUMULH. As with the integer Mul, the unpredicated encodings are SVE2 and the predicated siblings baseline SVE, so the tests gate on ARM64.SVE2(). For #79781. Change-Id: I52dcc8ba25410813035f187a757e40af766141f9 Reviewed-on: https://go-review.googlesource.com/c/go/+/822943 LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: David Chase Reviewed-by: Junyang Shao Auto-Submit: Junyang Shao --- src/cmd/compile/internal/arm64/simdssa_sve.go | 24 + .../internal/ssa/_gen/simdARM64SVE.rules | 40 + .../internal/ssa/_gen/simdARM64SVEops.go | 24 + .../internal/ssa/_gen/simdgenericOps.go | 8 + src/cmd/compile/internal/ssa/ssaop/opGen.go | 464 +++++++ .../internal/ssagen/simdARM64SVEintrinsics.go | 8 + .../ssarewrite/rewritearm64/rewriteARM64.go | 1184 ++++++++++++++++- .../archsimd/_gen/simdgen/ops/Mul/go_sve.yaml | 18 + .../simd_test/binary_sve_arm64_test.go | 28 + src/simd/archsimd/ops_sve.go | 42 + 10 files changed, 1816 insertions(+), 24 deletions(-) diff --git a/src/cmd/compile/internal/arm64/simdssa_sve.go b/src/cmd/compile/internal/arm64/simdssa_sve.go index d2ee3f9462b231..6a7db675fe78c2 100644 --- a/src/cmd/compile/internal/arm64/simdssa_sve.go +++ b/src/cmd/compile/internal/arm64/simdssa_sve.go @@ -47,6 +47,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDB, ssaop.OpARM64ZUQADDB, ssaop.OpARM64ZMULB, + ssaop.OpARM64ZSMULHB, + ssaop.OpARM64ZUMULHB, ssaop.OpARM64ZSUBB, ssaop.OpARM64ZSQSUBB, ssaop.OpARM64ZUQSUBB: @@ -60,6 +62,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZBICD, ssaop.OpARM64ZFMULD, ssaop.OpARM64ZMULD, + ssaop.OpARM64ZSMULHD, + ssaop.OpARM64ZUMULHD, ssaop.OpARM64ZORRD, ssaop.OpARM64ZFSUBD, ssaop.OpARM64ZSUBD, @@ -72,6 +76,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZSQADDH, ssaop.OpARM64ZUQADDH, ssaop.OpARM64ZMULH, + ssaop.OpARM64ZSMULHH, + ssaop.OpARM64ZUMULHH, ssaop.OpARM64ZSUBH, ssaop.OpARM64ZSQSUBH, ssaop.OpARM64ZUQSUBH: @@ -83,6 +89,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDS, ssaop.OpARM64ZFMULS, ssaop.OpARM64ZMULS, + ssaop.OpARM64ZSMULHS, + ssaop.OpARM64ZUMULHS, ssaop.OpARM64ZFSUBS, ssaop.OpARM64ZSUBS, ssaop.OpARM64ZSQSUBS, @@ -135,6 +143,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZANDMergingB, ssaop.OpARM64ZBICMergingB, ssaop.OpARM64ZMULMergingB, + ssaop.OpARM64ZSMULHMergingB, + ssaop.OpARM64ZUMULHMergingB, ssaop.OpARM64ZORRMergingB, ssaop.OpARM64ZSUBMergingB, ssaop.OpARM64ZSQSUBMergingB, @@ -150,6 +160,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZBICMergingD, ssaop.OpARM64ZFMULMergingD, ssaop.OpARM64ZMULMergingD, + ssaop.OpARM64ZSMULHMergingD, + ssaop.OpARM64ZUMULHMergingD, ssaop.OpARM64ZORRMergingD, ssaop.OpARM64ZFSUBMergingD, ssaop.OpARM64ZSUBMergingD, @@ -164,6 +176,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZANDMergingH, ssaop.OpARM64ZBICMergingH, ssaop.OpARM64ZMULMergingH, + ssaop.OpARM64ZSMULHMergingH, + ssaop.OpARM64ZUMULHMergingH, ssaop.OpARM64ZORRMergingH, ssaop.OpARM64ZSUBMergingH, ssaop.OpARM64ZSQSUBMergingH, @@ -179,6 +193,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZBICMergingS, ssaop.OpARM64ZFMULMergingS, ssaop.OpARM64ZMULMergingS, + ssaop.OpARM64ZSMULHMergingS, + ssaop.OpARM64ZUMULHMergingS, ssaop.OpARM64ZORRMergingS, ssaop.OpARM64ZFSUBMergingS, ssaop.OpARM64ZSUBMergingS, @@ -214,6 +230,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingPrefixedB, ssaop.OpARM64ZANDMergingPrefixedB, ssaop.OpARM64ZMULMergingPrefixedB, + ssaop.OpARM64ZSMULHMergingPrefixedB, + ssaop.OpARM64ZUMULHMergingPrefixedB, ssaop.OpARM64ZORRMergingPrefixedB, ssaop.OpARM64ZEORMergingPrefixedB: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_B) @@ -225,6 +243,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZANDMergingPrefixedD, ssaop.OpARM64ZFMULMergingPrefixedD, ssaop.OpARM64ZMULMergingPrefixedD, + ssaop.OpARM64ZSMULHMergingPrefixedD, + ssaop.OpARM64ZUMULHMergingPrefixedD, ssaop.OpARM64ZORRMergingPrefixedD, ssaop.OpARM64ZEORMergingPrefixedD: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_D) @@ -234,6 +254,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZUQADDMergingPrefixedH, ssaop.OpARM64ZANDMergingPrefixedH, ssaop.OpARM64ZMULMergingPrefixedH, + ssaop.OpARM64ZSMULHMergingPrefixedH, + ssaop.OpARM64ZUMULHMergingPrefixedH, ssaop.OpARM64ZORRMergingPrefixedH, ssaop.OpARM64ZEORMergingPrefixedH: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_H) @@ -245,6 +267,8 @@ func ssaGenSIMDSVEValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpARM64ZANDMergingPrefixedS, ssaop.OpARM64ZFMULMergingPrefixedS, ssaop.OpARM64ZMULMergingPrefixedS, + ssaop.OpARM64ZSMULHMergingPrefixedS, + ssaop.OpARM64ZUMULHMergingPrefixedS, ssaop.OpARM64ZORRMergingPrefixedS, ssaop.OpARM64ZEORMergingPrefixedS: p = simdZ3kvPredResultInArg0(s, v, arm64.ARNG_S) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules index 48600f8e556728..b77cee6c47af2c 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules @@ -110,6 +110,22 @@ (GreaterUint32s x y) => (ZCMPHIS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) (GreaterUint64s x y) => (ZCMPHID x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) (GreaterUint8s x y) => (ZCMPHIB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(MulHighInt16s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZSMULHH x y) +(MulHighInt16s x y) => (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) +(MulHighInt32s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZSMULHS x y) +(MulHighInt32s x y) => (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) +(MulHighInt64s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZSMULHD x y) +(MulHighInt64s x y) => (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) +(MulHighInt8s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZSMULHB x y) +(MulHighInt8s x y) => (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) +(MulHighUint16s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZUMULHH x y) +(MulHighUint16s x y) => (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) +(MulHighUint32s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZUMULHS x y) +(MulHighUint32s x y) => (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) +(MulHighUint64s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZUMULHD x y) +(MulHighUint64s x y) => (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) +(MulHighUint8s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZUMULHB x y) +(MulHighUint8s x y) => (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) (MulInt16s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULH x y) (MulInt16s x y) => (ZMULMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) (MulInt32s x y) && v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) => (ZMULS x y) @@ -166,11 +182,17 @@ (ZSELB (ZORRD x y) x mask) => (ZORRMergingB x y mask) (ZSELB (ZORRD x y) y mask) => (ZORRMergingB y x mask) (ZSELB (ZORRD x y) z mask) => (ZORRMergingPrefixedB z x y mask) +(ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) => (ZSMULHMergingB x y mask) +(ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) => (ZSMULHMergingB y x mask) +(ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) => (ZSMULHMergingPrefixedB z x y mask) (ZSELB (ZSQADDB x y) x mask) => (ZSQADDMergingB x y mask) (ZSELB (ZSQADDB x y) y mask) => (ZSQADDMergingB y x mask) (ZSELB (ZSQADDB x y) z mask) => (ZSQADDMergingPrefixedB z x y mask) (ZSELB (ZSQSUBB x y) x mask) => (ZSQSUBMergingB x y mask) (ZSELB (ZSUBB x y) x mask) => (ZSUBMergingB x y mask) +(ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) => (ZUMULHMergingB x y mask) +(ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) => (ZUMULHMergingB y x mask) +(ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) => (ZUMULHMergingPrefixedB z x y mask) (ZSELB (ZUQADDB x y) x mask) => (ZUQADDMergingB x y mask) (ZSELB (ZUQADDB x y) y mask) => (ZUQADDMergingB y x mask) (ZSELB (ZUQADDB x y) z mask) => (ZUQADDMergingPrefixedB z x y mask) @@ -203,11 +225,17 @@ (ZSELD (ZORRD x y) x mask) => (ZORRMergingD x y mask) (ZSELD (ZORRD x y) y mask) => (ZORRMergingD y x mask) (ZSELD (ZORRD x y) z mask) => (ZORRMergingPrefixedD z x y mask) +(ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) => (ZSMULHMergingD x y mask) +(ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) => (ZSMULHMergingD y x mask) +(ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZSMULHMergingPrefixedD z x y mask) (ZSELD (ZSQADDD x y) x mask) => (ZSQADDMergingD x y mask) (ZSELD (ZSQADDD x y) y mask) => (ZSQADDMergingD y x mask) (ZSELD (ZSQADDD x y) z mask) => (ZSQADDMergingPrefixedD z x y mask) (ZSELD (ZSQSUBD x y) x mask) => (ZSQSUBMergingD x y mask) (ZSELD (ZSUBD x y) x mask) => (ZSUBMergingD x y mask) +(ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) => (ZUMULHMergingD x y mask) +(ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) => (ZUMULHMergingD y x mask) +(ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) => (ZUMULHMergingPrefixedD z x y mask) (ZSELD (ZUQADDD x y) x mask) => (ZUQADDMergingD x y mask) (ZSELD (ZUQADDD x y) y mask) => (ZUQADDMergingD y x mask) (ZSELD (ZUQADDD x y) z mask) => (ZUQADDMergingPrefixedD z x y mask) @@ -230,11 +258,17 @@ (ZSELH (ZORRD x y) x mask) => (ZORRMergingH x y mask) (ZSELH (ZORRD x y) y mask) => (ZORRMergingH y x mask) (ZSELH (ZORRD x y) z mask) => (ZORRMergingPrefixedH z x y mask) +(ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) => (ZSMULHMergingH x y mask) +(ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) => (ZSMULHMergingH y x mask) +(ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) => (ZSMULHMergingPrefixedH z x y mask) (ZSELH (ZSQADDH x y) x mask) => (ZSQADDMergingH x y mask) (ZSELH (ZSQADDH x y) y mask) => (ZSQADDMergingH y x mask) (ZSELH (ZSQADDH x y) z mask) => (ZSQADDMergingPrefixedH z x y mask) (ZSELH (ZSQSUBH x y) x mask) => (ZSQSUBMergingH x y mask) (ZSELH (ZSUBH x y) x mask) => (ZSUBMergingH x y mask) +(ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) => (ZUMULHMergingH x y mask) +(ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) => (ZUMULHMergingH y x mask) +(ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) => (ZUMULHMergingPrefixedH z x y mask) (ZSELH (ZUQADDH x y) x mask) => (ZUQADDMergingH x y mask) (ZSELH (ZUQADDH x y) y mask) => (ZUQADDMergingH y x mask) (ZSELH (ZUQADDH x y) z mask) => (ZUQADDMergingPrefixedH z x y mask) @@ -267,11 +301,17 @@ (ZSELS (ZORRD x y) x mask) => (ZORRMergingS x y mask) (ZSELS (ZORRD x y) y mask) => (ZORRMergingS y x mask) (ZSELS (ZORRD x y) z mask) => (ZORRMergingPrefixedS z x y mask) +(ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) => (ZSMULHMergingS x y mask) +(ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) => (ZSMULHMergingS y x mask) +(ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZSMULHMergingPrefixedS z x y mask) (ZSELS (ZSQADDS x y) x mask) => (ZSQADDMergingS x y mask) (ZSELS (ZSQADDS x y) y mask) => (ZSQADDMergingS y x mask) (ZSELS (ZSQADDS x y) z mask) => (ZSQADDMergingPrefixedS z x y mask) (ZSELS (ZSQSUBS x y) x mask) => (ZSQSUBMergingS x y mask) (ZSELS (ZSUBS x y) x mask) => (ZSUBMergingS x y mask) +(ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) => (ZUMULHMergingS x y mask) +(ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) => (ZUMULHMergingS y x mask) +(ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) => (ZUMULHMergingPrefixedS z x y mask) (ZSELS (ZUQADDS x y) x mask) => (ZUQADDMergingS x y mask) (ZSELS (ZUQADDS x y) y mask) => (ZUQADDMergingS y x mask) (ZSELS (ZUQADDS x y) z mask) => (ZUQADDMergingPrefixedS z x y mask) diff --git a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go index 062387c5a18150..7a5ef0fe9ed9b7 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go +++ b/src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go @@ -144,6 +144,18 @@ func simdARM64SVEOps(z11, z21, zkv, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []op {name: "ZORRMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZORR", typ: "Vec256", resultInArg0: true}, {name: "ZORRMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZORR", typ: "Vec256", resultInArg0: true}, {name: "ZORRMergingS", argLength: 3, reg: z2kvPred, asm: "ZORR", commutative: true, typ: "Vec256"}, + {name: "ZSMULHB", argLength: 2, reg: z21, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHD", argLength: 2, reg: z21, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHH", argLength: 2, reg: z21, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHMergingB", argLength: 3, reg: z2kvPred, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHMergingD", argLength: 3, reg: z2kvPred, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHMergingH", argLength: 3, reg: z2kvPred, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZSMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZSMULHMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZSMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZSMULHMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZSMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZSMULHMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZSMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZSMULHMergingS", argLength: 3, reg: z2kvPred, asm: "ZSMULH", commutative: true, typ: "Vec256"}, + {name: "ZSMULHS", argLength: 2, reg: z21, asm: "ZSMULH", commutative: true, typ: "Vec256"}, {name: "ZSQADDB", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDD", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, {name: "ZSQADDH", argLength: 2, reg: z21, asm: "ZSQADD", commutative: true, typ: "Vec256"}, @@ -172,6 +184,18 @@ func simdARM64SVEOps(z11, z21, zkv, z2kk, z2kv, z2kvPred, z3kvPred regInfo) []op {name: "ZSUBMergingH", argLength: 3, reg: z2kvPred, asm: "ZSUB", typ: "Vec256", resultInArg0: true}, {name: "ZSUBMergingS", argLength: 3, reg: z2kvPred, asm: "ZSUB", typ: "Vec256", resultInArg0: true}, {name: "ZSUBS", argLength: 2, reg: z21, asm: "ZSUB", typ: "Vec256"}, + {name: "ZUMULHB", argLength: 2, reg: z21, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHD", argLength: 2, reg: z21, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHH", argLength: 2, reg: z21, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHMergingB", argLength: 3, reg: z2kvPred, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHMergingD", argLength: 3, reg: z2kvPred, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHMergingH", argLength: 3, reg: z2kvPred, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHMergingPrefixedB", argLength: 4, reg: z3kvPred, asm: "ZUMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZUMULHMergingPrefixedD", argLength: 4, reg: z3kvPred, asm: "ZUMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZUMULHMergingPrefixedH", argLength: 4, reg: z3kvPred, asm: "ZUMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZUMULHMergingPrefixedS", argLength: 4, reg: z3kvPred, asm: "ZUMULH", typ: "Vec256", resultInArg0: true}, + {name: "ZUMULHMergingS", argLength: 3, reg: z2kvPred, asm: "ZUMULH", commutative: true, typ: "Vec256"}, + {name: "ZUMULHS", argLength: 2, reg: z21, asm: "ZUMULH", commutative: true, typ: "Vec256"}, {name: "ZUQADDB", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDD", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, {name: "ZUQADDH", argLength: 2, reg: z21, asm: "ZUQADD", commutative: true, typ: "Vec256"}, diff --git a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go index 507b37a8663408..c940c192177b74 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go @@ -826,12 +826,20 @@ func simdGenericOps() []opData { {name: "MulFloat64x2", argLength: 2, commutative: true}, // ARCH:amd64,arm64,wasm {name: "MulFloat64x4", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulFloat64x8", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulHighInt8s", argLength: 2, commutative: true}, // ARCH:sve + {name: "MulHighInt16s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulHighInt16x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighInt16x16", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighInt16x32", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulHighInt32s", argLength: 2, commutative: true}, // ARCH:sve + {name: "MulHighInt64s", argLength: 2, commutative: true}, // ARCH:sve + {name: "MulHighUint8s", argLength: 2, commutative: true}, // ARCH:sve + {name: "MulHighUint16s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulHighUint16x8", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighUint16x16", argLength: 2, commutative: true}, // ARCH:amd64 {name: "MulHighUint16x32", argLength: 2, commutative: true}, // ARCH:amd64 + {name: "MulHighUint32s", argLength: 2, commutative: true}, // ARCH:sve + {name: "MulHighUint64s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt8s", argLength: 2, commutative: true}, // ARCH:sve {name: "MulInt8x16", argLength: 2, commutative: true}, // ARCH:arm64 {name: "MulInt16s", argLength: 2, commutative: true}, // ARCH:sve diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index 1701259349eb42..94de1d08740876 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -5180,6 +5180,18 @@ const ( OpARM64ZORRMergingPrefixedH OpARM64ZORRMergingPrefixedS OpARM64ZORRMergingS + OpARM64ZSMULHB + OpARM64ZSMULHD + OpARM64ZSMULHH + OpARM64ZSMULHMergingB + OpARM64ZSMULHMergingD + OpARM64ZSMULHMergingH + OpARM64ZSMULHMergingPrefixedB + OpARM64ZSMULHMergingPrefixedD + OpARM64ZSMULHMergingPrefixedH + OpARM64ZSMULHMergingPrefixedS + OpARM64ZSMULHMergingS + OpARM64ZSMULHS OpARM64ZSQADDB OpARM64ZSQADDD OpARM64ZSQADDH @@ -5208,6 +5220,18 @@ const ( OpARM64ZSUBMergingH OpARM64ZSUBMergingS OpARM64ZSUBS + OpARM64ZUMULHB + OpARM64ZUMULHD + OpARM64ZUMULHH + OpARM64ZUMULHMergingB + OpARM64ZUMULHMergingD + OpARM64ZUMULHMergingH + OpARM64ZUMULHMergingPrefixedB + OpARM64ZUMULHMergingPrefixedD + OpARM64ZUMULHMergingPrefixedH + OpARM64ZUMULHMergingPrefixedS + OpARM64ZUMULHMergingS + OpARM64ZUMULHS OpARM64ZUQADDB OpARM64ZUQADDD OpARM64ZUQADDH @@ -8022,12 +8046,20 @@ const ( OpMulFloat64x2 OpMulFloat64x4 OpMulFloat64x8 + OpMulHighInt16s OpMulHighInt16x16 OpMulHighInt16x32 OpMulHighInt16x8 + OpMulHighInt32s + OpMulHighInt64s + OpMulHighInt8s + OpMulHighUint16s OpMulHighUint16x16 OpMulHighUint16x32 OpMulHighUint16x8 + OpMulHighUint32s + OpMulHighUint64s + OpMulHighUint8s OpMulInt16s OpMulInt16x16 OpMulInt16x32 @@ -88117,6 +88149,198 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZSMULHB", + ArgLen: 2, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHD", + ArgLen: 2, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHH", + ArgLen: 2, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZSMULHS", + ArgLen: 2, + Commutative: true, + asm: arm64.AZSMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZSQADDB", ArgLen: 2, @@ -88549,6 +88773,198 @@ var OpcodeTable = [...]OpInfo{ }, }, }, + { + Name: "ZUMULHB", + ArgLen: 2, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHD", + ArgLen: 2, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHH", + ArgLen: 2, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingB", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingD", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingH", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingPrefixedB", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingPrefixedD", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingPrefixedH", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingPrefixedS", + ArgLen: 4, + ResultInArg0: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {3, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {2, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHMergingS", + ArgLen: 3, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {2, RegMask{V1: 9223372036854775808, V2: 32767}}, // P0 P1 P2 P3 P4 P5 P6 P7 P8 P9 P10 P11 P12 P13 P14 P15 + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, + { + Name: "ZUMULHS", + ArgLen: 2, + Commutative: true, + asm: arm64.AZUMULH, + Reg: RegInfo{ + Inputs: []InputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + {1, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + Outputs: []OutputInfo{ + {0, RegMask{V1: 9223372034707292160, V2: 0}}, // F0 F1 F2 F3 F4 F5 F6 F7 F8 F9 F10 F11 F12 F13 F14 F15 F16 F17 F18 F19 F20 F21 F22 F23 F24 F25 F26 F27 F28 F29 F30 F31 + }, + }, + }, { Name: "ZUQADDB", ArgLen: 2, @@ -116988,6 +117404,12 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulHighInt16s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulHighInt16x16", ArgLen: 2, @@ -117006,6 +117428,30 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulHighInt32s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, + { + Name: "MulHighInt64s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, + { + Name: "MulHighInt8s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, + { + Name: "MulHighUint16s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulHighUint16x16", ArgLen: 2, @@ -117024,6 +117470,24 @@ var OpcodeTable = [...]OpInfo{ Commutative: true, Generic: true, }, + { + Name: "MulHighUint32s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, + { + Name: "MulHighUint64s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, + { + Name: "MulHighUint8s", + ArgLen: 2, + Commutative: true, + Generic: true, + }, { Name: "MulInt16s", ArgLen: 2, diff --git a/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go b/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go index f9111340306048..6c091da544778b 100644 --- a/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go +++ b/src/cmd/compile/internal/ssagen/simdARM64SVEintrinsics.go @@ -95,6 +95,14 @@ func simdARM64SVEIntrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFa addF(simdPackage, "Uint16s.Mul", opLen2(ssaop.OpMulUint16s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Uint32s.Mul", opLen2(ssaop.OpMulUint32s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Uint64s.Mul", opLen2(ssaop.OpMulUint64s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int8s.MulHigh", opLen2(ssaop.OpMulHighInt8s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int16s.MulHigh", opLen2(ssaop.OpMulHighInt16s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int32s.MulHigh", opLen2(ssaop.OpMulHighInt32s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Int64s.MulHigh", opLen2(ssaop.OpMulHighInt64s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint8s.MulHigh", opLen2(ssaop.OpMulHighUint8s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint16s.MulHigh", opLen2(ssaop.OpMulHighUint16s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint32s.MulHigh", opLen2(ssaop.OpMulHighUint32s, types.TypeVec256), sys.ARM64) + addF(simdPackage, "Uint64s.MulHigh", opLen2(ssaop.OpMulHighUint64s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Float32s.Neg", opLen1(ssaop.OpNegFloat32s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Float64s.Neg", opLen1(ssaop.OpNegFloat64s, types.TypeVec256), sys.ARM64) addF(simdPackage, "Int8s.Neg", opLen1(ssaop.OpNegInt8s, types.TypeVec256), sys.ARM64) diff --git a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go index a3d8ea9de1035c..0f03956ef79c9b 100644 --- a/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go +++ b/src/cmd/compile/internal/ssarewrite/rewritearm64/rewriteARM64.go @@ -1853,6 +1853,22 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpMulFloat64x2: v.Op = ssaop.OpARM64VFMUL2D return true + case ssaop.OpMulHighInt16s: + return rewriteValue_OpMulHighInt16s(v) + case ssaop.OpMulHighInt32s: + return rewriteValue_OpMulHighInt32s(v) + case ssaop.OpMulHighInt64s: + return rewriteValue_OpMulHighInt64s(v) + case ssaop.OpMulHighInt8s: + return rewriteValue_OpMulHighInt8s(v) + case ssaop.OpMulHighUint16s: + return rewriteValue_OpMulHighUint16s(v) + case ssaop.OpMulHighUint32s: + return rewriteValue_OpMulHighUint32s(v) + case ssaop.OpMulHighUint64s: + return rewriteValue_OpMulHighUint64s(v) + case ssaop.OpMulHighUint8s: + return rewriteValue_OpMulHighUint8s(v) case ssaop.OpMulInt16s: return rewriteValue_OpMulInt16s(v) case ssaop.OpMulInt16x8: @@ -20546,6 +20562,110 @@ func rewriteValue_OpARM64ZSELB(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) + // result: (ZSMULHMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) + // result: (ZSMULHMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) + // result: (ZSMULHMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingB { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELB (ZSQADDB x y) x mask) // result: (ZSQADDMergingB x y mask) for { @@ -20636,6 +20756,110 @@ func rewriteValue_OpARM64ZSELB(v *ssa.Value) bool { v.AddArg3(x, y, mask) return true } + // match: (ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) x mask) + // result: (ZUMULHMergingB x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingB) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) y mask) + // result: (ZUMULHMergingB y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingB { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingB) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELB (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) z mask) + // result: (ZUMULHMergingPrefixedB z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingB { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTB { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 32 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingPrefixedB) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELB (ZUQADDB x y) x mask) // result: (ZUQADDMergingB x y mask) for { @@ -21355,66 +21579,170 @@ func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } - // match: (ZSELD (ZSQADDD x y) x mask) - // result: (ZSQADDMergingD x y mask) + // match: (ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) + // result: (ZSMULHMergingD x y mask) for { - if v_0.Op != ssaop.OpARM64ZSQADDD { + if v_0.Op != ssaop.OpARM64ZSMULHMergingD { break } - _ = v_0.Args[1] + _ = v_0.Args[2] v_0_0 := v_0.Args[0] v_0_1 := v_0.Args[1] for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { x := v_0_0 y := v_0_1 - if x != v_1 { + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || x != v_1 { continue } mask := v_2 - v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.Reset(ssaop.OpARM64ZSMULHMergingD) v.AddArg3(x, y, mask) return true } break } - // match: (ZSELD (ZSQADDD x y) y mask) - // result: (ZSQADDMergingD y x mask) + // match: (ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) + // result: (ZSMULHMergingD y x mask) for { - if v_0.Op != ssaop.OpARM64ZSQADDD { + if v_0.Op != ssaop.OpARM64ZSMULHMergingD { break } - _ = v_0.Args[1] + _ = v_0.Args[2] v_0_0 := v_0.Args[0] v_0_1 := v_0.Args[1] for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { x := v_0_0 y := v_0_1 - if y != v_1 { + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || y != v_1 { continue } mask := v_2 - v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.Reset(ssaop.OpARM64ZSMULHMergingD) v.AddArg3(y, x, mask) return true } break } - // match: (ZSELD (ZSQADDD x y) z mask) - // result: (ZSQADDMergingPrefixedD z x y mask) + // match: (ZSELD (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) + // result: (ZSMULHMergingPrefixedD z x y mask) for { - if v_0.Op != ssaop.OpARM64ZSQADDD { + if v_0.Op != ssaop.OpARM64ZSMULHMergingD { break } - y := v_0.Args[1] + _ = v_0.Args[2] x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 { + break + } z := v_1 mask := v_2 - v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedD) + v.Reset(ssaop.OpARM64ZSMULHMergingPrefixedD) v.AddArg4(z, x, y, mask) return true } - // match: (ZSELD (ZSQSUBD x y) x mask) - // result: (ZSQSUBMergingD x y mask) + // match: (ZSELD (ZSQADDD x y) x mask) + // result: (ZSQADDMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) y mask) + // result: (ZSQADDMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + _ = v_0.Args[1] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + if y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZSQADDD x y) z mask) + // result: (ZSQADDMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSQADDD { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSQADDMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } + // match: (ZSELD (ZSQSUBD x y) x mask) + // result: (ZSQSUBMergingD x y mask) for { if v_0.Op != ssaop.OpARM64ZSQSUBD { break @@ -21445,6 +21773,110 @@ func rewriteValue_OpARM64ZSELD(v *ssa.Value) bool { v.AddArg3(x, y, mask) return true } + // match: (ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) x mask) + // result: (ZUMULHMergingD x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingD { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingD) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) y mask) + // result: (ZUMULHMergingD y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingD { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingD) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELD (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) z mask) + // result: (ZUMULHMergingPrefixedD z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingD { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTD { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 4 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingPrefixedD) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELD (ZUQADDD x y) x mask) // result: (ZUQADDMergingD x y mask) for { @@ -21939,6 +22371,110 @@ func rewriteValue_OpARM64ZSELH(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) + // result: (ZSMULHMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) + // result: (ZSMULHMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) + // result: (ZSMULHMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingH { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELH (ZSQADDH x y) x mask) // result: (ZSQADDMergingH x y mask) for { @@ -22029,6 +22565,110 @@ func rewriteValue_OpARM64ZSELH(v *ssa.Value) bool { v.AddArg3(x, y, mask) return true } + // match: (ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) x mask) + // result: (ZUMULHMergingH x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingH) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) y mask) + // result: (ZUMULHMergingH y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingH { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingH) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELH (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) z mask) + // result: (ZUMULHMergingPrefixedH z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingH { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTH { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 16 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingPrefixedH) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELH (ZUQADDH x y) x mask) // result: (ZUQADDMergingH x y mask) for { @@ -22748,6 +23388,110 @@ func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { v.AddArg4(z, x, y, mask) return true } + // match: (ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) + // result: (ZSMULHMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) + // result: (ZSMULHMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) + // result: (ZSMULHMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZSMULHMergingS { + break + } + _ = v_0.Args[2] + x := v_0.Args[0] + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + break + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 { + break + } + z := v_1 + mask := v_2 + v.Reset(ssaop.OpARM64ZSMULHMergingPrefixedS) + v.AddArg4(z, x, y, mask) + return true + } // match: (ZSELS (ZSQADDS x y) x mask) // result: (ZSQADDMergingS x y mask) for { @@ -22825,17 +23569,121 @@ func rewriteValue_OpARM64ZSELS(v *ssa.Value) bool { // match: (ZSELS (ZSUBS x y) x mask) // result: (ZSUBMergingS x y mask) for { - if v_0.Op != ssaop.OpARM64ZSUBS { + if v_0.Op != ssaop.OpARM64ZSUBS { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + if x != v_1 { + break + } + mask := v_2 + v.Reset(ssaop.OpARM64ZSUBMergingS) + v.AddArg3(x, y, mask) + return true + } + // match: (ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) x mask) + // result: (ZUMULHMergingS x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || x != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingS) + v.AddArg3(x, y, mask) + return true + } + break + } + // match: (ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) y mask) + // result: (ZUMULHMergingS y x mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingS { + break + } + _ = v_0.Args[2] + v_0_0 := v_0.Args[0] + v_0_1 := v_0.Args[1] + for _i0 := 0; _i0 <= 1; _i0, v_0_0, v_0_1 = _i0+1, v_0_1, v_0_0 { + x := v_0_0 + y := v_0_1 + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + continue + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + continue + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { + continue + } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 || y != v_1 { + continue + } + mask := v_2 + v.Reset(ssaop.OpARM64ZUMULHMergingS) + v.AddArg3(y, x, mask) + return true + } + break + } + // match: (ZSELS (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) z mask) + // result: (ZUMULHMergingPrefixedS z x y mask) + for { + if v_0.Op != ssaop.OpARM64ZUMULHMergingS { break } - y := v_0.Args[1] + _ = v_0.Args[2] x := v_0.Args[0] - if x != v_1 { + y := v_0.Args[1] + v_0_2 := v_0.Args[2] + if v_0_2.Op != ssaop.OpSelect0 || v_0_2.Type != types.TypeMask { + break + } + v_0_2_0 := v_0_2.Args[0] + if v_0_2_0.Op != ssaop.OpARM64PWHILELTS { + break + } + _ = v_0_2_0.Args[1] + v_0_2_0_0 := v_0_2_0.Args[0] + if v_0_2_0_0.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_0.AuxInt) != 0 { break } + v_0_2_0_1 := v_0_2_0.Args[1] + if v_0_2_0_1.Op != ssaop.OpARM64MOVDconst || ssa.AuxIntToInt64(v_0_2_0_1.AuxInt) != 8 { + break + } + z := v_1 mask := v_2 - v.Reset(ssaop.OpARM64ZSUBMergingS) - v.AddArg3(x, y, mask) + v.Reset(ssaop.OpARM64ZUMULHMergingPrefixedS) + v.AddArg4(z, x, y, mask) return true } // match: (ZSELS (ZUQADDS x y) x mask) @@ -26429,6 +27277,294 @@ func rewriteValue_OpMulAddUint8x16(v *ssa.Value) bool { return true } } +func rewriteValue_OpMulHighInt16s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighInt16s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZSMULHH x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZSMULHH) + v.AddArg2(x, y) + return true + } + // match: (MulHighInt16s x y) + // result: (ZSMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZSMULHMergingH) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTH, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(16) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighInt32s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighInt32s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZSMULHS x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZSMULHS) + v.AddArg2(x, y) + return true + } + // match: (MulHighInt32s x y) + // result: (ZSMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZSMULHMergingS) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTS, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(8) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighInt64s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighInt64s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZSMULHD x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZSMULHD) + v.AddArg2(x, y) + return true + } + // match: (MulHighInt64s x y) + // result: (ZSMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZSMULHMergingD) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTD, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(4) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighInt8s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighInt8s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZSMULHB x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZSMULHB) + v.AddArg2(x, y) + return true + } + // match: (MulHighInt8s x y) + // result: (ZSMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZSMULHMergingB) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTB, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(32) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighUint16s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighUint16s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZUMULHH x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZUMULHH) + v.AddArg2(x, y) + return true + } + // match: (MulHighUint16s x y) + // result: (ZUMULHMergingH x y (Select0 (PWHILELTH (MOVDconst [0]) (MOVDconst [16])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZUMULHMergingH) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTH, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(16) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighUint32s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighUint32s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZUMULHS x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZUMULHS) + v.AddArg2(x, y) + return true + } + // match: (MulHighUint32s x y) + // result: (ZUMULHMergingS x y (Select0 (PWHILELTS (MOVDconst [0]) (MOVDconst [8])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZUMULHMergingS) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTS, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(8) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighUint64s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighUint64s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZUMULHD x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZUMULHD) + v.AddArg2(x, y) + return true + } + // match: (MulHighUint64s x y) + // result: (ZUMULHMergingD x y (Select0 (PWHILELTD (MOVDconst [0]) (MOVDconst [4])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZUMULHMergingD) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTD, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(4) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} +func rewriteValue_OpMulHighUint8s(v *ssa.Value) bool { + v_1 := v.Args[1] + v_0 := v.Args[0] + b := v.Block + typ := &b.Func.Config.Types + // match: (MulHighUint8s x y) + // cond: v.Block.CPUfeatures.HasFeature(ssa.CPUsve2) + // result: (ZUMULHB x y) + for { + x := v_0 + y := v_1 + if !(v.Block.CPUfeatures.HasFeature(ssa.CPUsve2)) { + break + } + v.Reset(ssaop.OpARM64ZUMULHB) + v.AddArg2(x, y) + return true + } + // match: (MulHighUint8s x y) + // result: (ZUMULHMergingB x y (Select0 (PWHILELTB (MOVDconst [0]) (MOVDconst [32])))) + for { + x := v_0 + y := v_1 + v.Reset(ssaop.OpARM64ZUMULHMergingB) + v0 := b.NewValue0(v.Pos, ssaop.OpSelect0, types.TypeMask) + v1 := b.NewValue0(v.Pos, ssaop.OpARM64PWHILELTB, types.NewTuple(typ.Mask, types.TypeFlags)) + v2 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v2.AuxInt = ssa.Int64ToAuxInt(0) + v3 := b.NewValue0(v.Pos, ssaop.OpARM64MOVDconst, typ.UInt64) + v3.AuxInt = ssa.Int64ToAuxInt(32) + v1.AddArg2(v2, v3) + v0.AddArg(v1) + v.AddArg3(x, y, v0) + return true + } +} func rewriteValue_OpMulInt16s(v *ssa.Value) bool { v_1 := v.Args[1] v_0 := v.Args[0] diff --git a/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml b/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml index a086bd129af86b..062f9a5a223efb 100644 --- a/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml +++ b/src/simd/archsimd/_gen/simdgen/ops/Mul/go_sve.yaml @@ -31,3 +31,21 @@ - *float out: - *float + +# MulHigh — integer multiply returning the high half, ZSMULH/ZUMULH. Like ZMUL, +# the unpredicated encodings are SVE2 and the predicated siblings baseline SVE. +- go: MulHigh + asm: "ZSMULH" + in: + - *int + - *int + out: + - *int + +- go: MulHigh + asm: "ZUMULH" + in: + - *uint + - *uint + out: + - *uint diff --git a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go index cbfee86b87129d..4d6a73edd365df 100644 --- a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go @@ -362,3 +362,31 @@ func TestMulSVE(t *testing.T) { testInt8sBinary(t, archsimd.Int8s.Mul, mulInt8) } } + +func TestMulHighSVE(t *testing.T) { + if !archsimd.ARM64.SVE() { + t.Skip("no SVE") + } + mulHighInt8 := func(x, y []int8) []int8 { + r := make([]int8, len(x)) + for i := range x { + r[i] = int8((int16(x[i]) * int16(y[i])) >> 8) + } + return r + } + mulHighUint8 := func(x, y []uint8) []uint8 { + r := make([]uint8, len(x)) + for i := range x { + r[i] = uint8((uint16(x[i]) * uint16(y[i])) >> 8) + } + return r + } + // Ungated: compiles to the merging-predicated fallback, correct on any SVE. + testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighInt8) + testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighUint8) + if archsimd.ARM64.SVE2() { + // Gated: this block compiles to the unpredicated SVE2 encodings. + testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighInt8) + testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighUint8) + } +} diff --git a/src/simd/archsimd/ops_sve.go b/src/simd/archsimd/ops_sve.go index d3e70668f2b334..bde69f8dcd99b7 100644 --- a/src/simd/archsimd/ops_sve.go +++ b/src/simd/archsimd/ops_sve.go @@ -446,6 +446,48 @@ func (x Uint32s) Mul(y Uint32s) Uint32s // Asm: ZMUL, CPU Feature: SVE func (x Uint64s) Mul(y Uint64s) Uint64s +/* MulHigh */ + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZSMULH, CPU Feature: SVE +func (x Int8s) MulHigh(y Int8s) Int8s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZSMULH, CPU Feature: SVE +func (x Int16s) MulHigh(y Int16s) Int16s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZSMULH, CPU Feature: SVE +func (x Int32s) MulHigh(y Int32s) Int32s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZSMULH, CPU Feature: SVE +func (x Int64s) MulHigh(y Int64s) Int64s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZUMULH, CPU Feature: SVE +func (x Uint8s) MulHigh(y Uint8s) Uint8s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZUMULH, CPU Feature: SVE +func (x Uint16s) MulHigh(y Uint16s) Uint16s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZUMULH, CPU Feature: SVE +func (x Uint32s) MulHigh(y Uint32s) Uint32s + +// MulHigh multiplies elements and stores the high part of the result. +// +// Asm: ZUMULH, CPU Feature: SVE +func (x Uint64s) MulHigh(y Uint64s) Uint64s + /* Neg */ // Neg returns the elementwise negation of x. From 8b1c22e12f478d4dfc7f7a390e92fbc68085b87e Mon Sep 17 00:00:00 2001 From: Junyang Shao Date: Thu, 3 Sep 2026 15:56:17 -0400 Subject: [PATCH 5/5] simd/archsimd: test SVE ops across all element shapes The SVE op tests spot-checked one or two element shapes each. Drive every op through every shape it is defined on, mirroring the Add/Sub pattern: generic emulations (addSaturated, subSaturated, mulHigh, negSlice, absSlice) join the existing slice helpers, and the per-shape driver wrappers are filled in for the missing widths. For #79781. Change-Id: I3ba5d48b6fe8ee1f3486593d913692485cf4c8f4 Reviewed-on: https://go-review.googlesource.com/c/go/+/827484 Auto-Submit: Junyang Shao LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com Reviewed-by: David Chase Reviewed-by: Junyang Shao --- .../simd_test/binary_sve_arm64_test.go | 347 ++++++++---------- .../simd_test/compare_sve_arm64_test.go | 84 ++++- .../internal/simd_test/simd_arm64_test.go | 45 +-- .../simd_test/simulation_helpers_test.go | 115 ++++++ 4 files changed, 360 insertions(+), 231 deletions(-) diff --git a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go index 4d6a73edd365df..447e35b89c2269 100644 --- a/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/binary_sve_arm64_test.go @@ -14,7 +14,6 @@ package simd_test import ( - "math" "simd/archsimd" "testing" ) @@ -77,6 +76,71 @@ func testFloat64sBinary(t *testing.T, f func(_, _ archsimd.Float64s) archsimd.Fl testSVEBinary(t, float64s, 8, z.Len(), archsimd.LoadFloat64s, f, archsimd.Float64s.Store, want) } +func testUint16sBinary(t *testing.T, f func(_, _ archsimd.Uint16s) archsimd.Uint16s, want func(_, _ []uint16) []uint16) { + var z archsimd.Uint16s + testSVEBinary(t, uint16s, 2, z.Len(), archsimd.LoadUint16s, f, archsimd.Uint16s.Store, want) +} + +func testUint32sBinary(t *testing.T, f func(_, _ archsimd.Uint32s) archsimd.Uint32s, want func(_, _ []uint32) []uint32) { + var z archsimd.Uint32s + testSVEBinary(t, uint32s, 4, z.Len(), archsimd.LoadUint32s, f, archsimd.Uint32s.Store, want) +} + +func testUint64sBinary(t *testing.T, f func(_, _ archsimd.Uint64s) archsimd.Uint64s, want func(_, _ []uint64) []uint64) { + var z archsimd.Uint64s + testSVEBinary(t, uint64s, 8, z.Len(), archsimd.LoadUint64s, f, archsimd.Uint64s.Store, want) +} + +func testInt8sUnary(t *testing.T, f func(archsimd.Int8s) archsimd.Int8s, want func([]int8) []int8) { + var z archsimd.Int8s + testSVEUnary(t, int8s, 1, z.Len(), archsimd.LoadInt8s, f, archsimd.Int8s.Store, want) +} + +func testInt16sUnary(t *testing.T, f func(archsimd.Int16s) archsimd.Int16s, want func([]int16) []int16) { + var z archsimd.Int16s + testSVEUnary(t, int16s, 2, z.Len(), archsimd.LoadInt16s, f, archsimd.Int16s.Store, want) +} + +func testInt32sUnary(t *testing.T, f func(archsimd.Int32s) archsimd.Int32s, want func([]int32) []int32) { + var z archsimd.Int32s + testSVEUnary(t, int32s, 4, z.Len(), archsimd.LoadInt32s, f, archsimd.Int32s.Store, want) +} + +func testInt64sUnary(t *testing.T, f func(archsimd.Int64s) archsimd.Int64s, want func([]int64) []int64) { + var z archsimd.Int64s + testSVEUnary(t, int64s, 8, z.Len(), archsimd.LoadInt64s, f, archsimd.Int64s.Store, want) +} + +func testUint8sUnary(t *testing.T, f func(archsimd.Uint8s) archsimd.Uint8s, want func([]uint8) []uint8) { + var z archsimd.Uint8s + testSVEUnary(t, uint8s, 1, z.Len(), archsimd.LoadUint8s, f, archsimd.Uint8s.Store, want) +} + +func testUint16sUnary(t *testing.T, f func(archsimd.Uint16s) archsimd.Uint16s, want func([]uint16) []uint16) { + var z archsimd.Uint16s + testSVEUnary(t, uint16s, 2, z.Len(), archsimd.LoadUint16s, f, archsimd.Uint16s.Store, want) +} + +func testUint32sUnary(t *testing.T, f func(archsimd.Uint32s) archsimd.Uint32s, want func([]uint32) []uint32) { + var z archsimd.Uint32s + testSVEUnary(t, uint32s, 4, z.Len(), archsimd.LoadUint32s, f, archsimd.Uint32s.Store, want) +} + +func testUint64sUnary(t *testing.T, f func(archsimd.Uint64s) archsimd.Uint64s, want func([]uint64) []uint64) { + var z archsimd.Uint64s + testSVEUnary(t, uint64s, 8, z.Len(), archsimd.LoadUint64s, f, archsimd.Uint64s.Store, want) +} + +func testFloat32sUnary(t *testing.T, f func(archsimd.Float32s) archsimd.Float32s, want func([]float32) []float32) { + var z archsimd.Float32s + testSVEUnary(t, float32s, 4, z.Len(), archsimd.LoadFloat32s, f, archsimd.Float32s.Store, want) +} + +func testFloat64sUnary(t *testing.T, f func(archsimd.Float64s) archsimd.Float64s, want func([]float64) []float64) { + var z archsimd.Float64s + testSVEUnary(t, float64s, 8, z.Len(), archsimd.LoadFloat64s, f, archsimd.Float64s.Store, want) +} + func TestAddSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") @@ -86,6 +150,9 @@ func TestAddSVE(t *testing.T) { testInt32sBinary(t, archsimd.Int32s.Add, addSlice[int32]) testInt64sBinary(t, archsimd.Int64s.Add, addSlice[int64]) testUint8sBinary(t, archsimd.Uint8s.Add, addSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Add, addSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Add, addSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Add, addSlice[uint64]) testFloat32sBinary(t, archsimd.Float32s.Add, addSlice[float32]) testFloat64sBinary(t, archsimd.Float64s.Add, addSlice[float64]) } @@ -99,6 +166,9 @@ func TestSubSVE(t *testing.T) { testInt32sBinary(t, archsimd.Int32s.Sub, subSlice[int32]) testInt64sBinary(t, archsimd.Int64s.Sub, subSlice[int64]) testUint8sBinary(t, archsimd.Uint8s.Sub, subSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Sub, subSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Sub, subSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Sub, subSlice[uint64]) testFloat32sBinary(t, archsimd.Float32s.Sub, subSlice[float32]) testFloat64sBinary(t, archsimd.Float64s.Sub, subSlice[float64]) } @@ -125,241 +195,148 @@ func TestAbsSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - absSlice := func(x []int8) []int8 { - r := make([]int8, len(x)) - for i, v := range x { - if v < 0 { - v = -v // -128 stays -128, as ABS does - } - r[i] = v - } - return r - } - var z archsimd.Int8s - testSVEUnary(t, int8s, 1, z.Len(), archsimd.LoadInt8s, archsimd.Int8s.Abs, archsimd.Int8s.Store, absSlice) - absFloat32 := func(x []float32) []float32 { - r := make([]float32, len(x)) - for i, v := range x { - r[i] = float32(math.Abs(float64(v))) - } - return r - } - var zf archsimd.Float32s - testSVEUnary(t, float32s, 4, zf.Len(), archsimd.LoadFloat32s, archsimd.Float32s.Abs, archsimd.Float32s.Store, absFloat32) + testInt8sUnary(t, archsimd.Int8s.Abs, absSlice[int8]) + testInt16sUnary(t, archsimd.Int16s.Abs, absSlice[int16]) + testInt32sUnary(t, archsimd.Int32s.Abs, absSlice[int32]) + testInt64sUnary(t, archsimd.Int64s.Abs, absSlice[int64]) + testFloat32sUnary(t, archsimd.Float32s.Abs, absSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Abs, absSlice[float64]) } func TestNegSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - negInt8 := func(x []int8) []int8 { - r := make([]int8, len(x)) - for i, v := range x { - r[i] = -v - } - return r - } - negFloat64 := func(x []float64) []float64 { - r := make([]float64, len(x)) - for i, v := range x { - r[i] = -v - } - return r - } - var zi archsimd.Int8s - testSVEUnary(t, int8s, 1, zi.Len(), archsimd.LoadInt8s, archsimd.Int8s.Neg, archsimd.Int8s.Store, negInt8) - var zf archsimd.Float64s - testSVEUnary(t, float64s, 8, zf.Len(), archsimd.LoadFloat64s, archsimd.Float64s.Neg, archsimd.Float64s.Store, negFloat64) + testInt8sUnary(t, archsimd.Int8s.Neg, negSlice[int8]) + testInt16sUnary(t, archsimd.Int16s.Neg, negSlice[int16]) + testInt32sUnary(t, archsimd.Int32s.Neg, negSlice[int32]) + testInt64sUnary(t, archsimd.Int64s.Neg, negSlice[int64]) + testFloat32sUnary(t, archsimd.Float32s.Neg, negSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Neg, negSlice[float64]) } func TestSqrtSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - var in, got [4]float64 - for i := range in { - in[i] = float64(i + 1) - } - v := archsimd.LoadFloat64s(in[:]) - v.Sqrt().Store(got[:]) - var z archsimd.Float64s - for i := 0; i < z.Len(); i++ { - if want := math.Sqrt(in[i]); got[i] != want { - t.Errorf("lane %d: Sqrt(%v) = %v, want %v", i, in[i], got[i], want) - } - } + testFloat32sUnary(t, archsimd.Float32s.Sqrt, sqrtSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Sqrt, sqrtSlice[float64]) } func TestCeilSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - var in, got [4]float64 - for i := range in { - in[i] = float64(i) - 1.5 - } - v := archsimd.LoadFloat64s(in[:]) - v.Ceil().Store(got[:]) - var z archsimd.Float64s - for i := 0; i < z.Len(); i++ { - if want := math.Ceil(in[i]); got[i] != want { - t.Errorf("lane %d: Ceil(%v) = %v, want %v", i, in[i], got[i], want) - } - } + testFloat32sUnary(t, archsimd.Float32s.Ceil, ceilSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Ceil, ceilSlice[float64]) } func TestFloorSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - var in, got [4]float64 - for i := range in { - in[i] = float64(i) - 1.5 - } - v := archsimd.LoadFloat64s(in[:]) - v.Floor().Store(got[:]) - var z archsimd.Float64s - for i := 0; i < z.Len(); i++ { - if want := math.Floor(in[i]); got[i] != want { - t.Errorf("lane %d: Floor(%v) = %v, want %v", i, in[i], got[i], want) - } - } + testFloat32sUnary(t, archsimd.Float32s.Floor, floorSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Floor, floorSlice[float64]) } func TestTruncSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - var in, got [4]float64 - for i := range in { - in[i] = float64(i) - 1.5 - } - v := archsimd.LoadFloat64s(in[:]) - v.Trunc().Store(got[:]) - var z archsimd.Float64s - for i := 0; i < z.Len(); i++ { - if want := math.Trunc(in[i]); got[i] != want { - t.Errorf("lane %d: Trunc(%v) = %v, want %v", i, in[i], got[i], want) - } - } + testFloat32sUnary(t, archsimd.Float32s.Trunc, truncSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Trunc, truncSlice[float64]) } func TestRoundSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - var in, got [4]float64 - for i := range in { - in[i] = float64(i) - 1.5 - } - v := archsimd.LoadFloat64s(in[:]) - v.Round().Store(got[:]) - var z archsimd.Float64s - for i := 0; i < z.Len(); i++ { - if want := math.RoundToEven(in[i]); got[i] != want { - t.Errorf("lane %d: Round(%v) = %v, want %v", i, in[i], got[i], want) - } - } + testFloat32sUnary(t, archsimd.Float32s.Round, roundSlice[float32]) + testFloat64sUnary(t, archsimd.Float64s.Round, roundSlice[float64]) } func TestAndSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - andInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = x[i] & y[i] - } - return r - } - testInt8sBinary(t, archsimd.Int8s.And, andInt8) - andUint64 := func(x, y []uint64) []uint64 { - r := make([]uint64, len(x)) - for i := range x { - r[i] = x[i] & y[i] - } - return r - } - var z archsimd.Uint64s - testSVEBinary(t, uint64s, 8, z.Len(), archsimd.LoadUint64s, archsimd.Uint64s.And, archsimd.Uint64s.Store, andUint64) + testInt8sBinary(t, archsimd.Int8s.And, andSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.And, andSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.And, andSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.And, andSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.And, andSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.And, andSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.And, andSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.And, andSlice[uint64]) } func TestOrSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - orInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = x[i] | y[i] - } - return r - } - testInt8sBinary(t, archsimd.Int8s.Or, orInt8) - orUint64 := func(x, y []uint64) []uint64 { - r := make([]uint64, len(x)) - for i := range x { - r[i] = x[i] | y[i] - } - return r - } - var z archsimd.Uint64s - testSVEBinary(t, uint64s, 8, z.Len(), archsimd.LoadUint64s, archsimd.Uint64s.Or, archsimd.Uint64s.Store, orUint64) + testInt8sBinary(t, archsimd.Int8s.Or, orSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.Or, orSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.Or, orSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.Or, orSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.Or, orSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Or, orSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Or, orSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Or, orSlice[uint64]) } func TestXorSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - xorInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = x[i] ^ y[i] - } - return r - } - testInt8sBinary(t, archsimd.Int8s.Xor, xorInt8) + testInt8sBinary(t, archsimd.Int8s.Xor, xorSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.Xor, xorSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.Xor, xorSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.Xor, xorSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.Xor, xorSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Xor, xorSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Xor, xorSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Xor, xorSlice[uint64]) } func TestAndNotSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - andNotInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = x[i] &^ y[i] - } - return r - } - testInt8sBinary(t, archsimd.Int8s.AndNot, andNotInt8) + testInt8sBinary(t, archsimd.Int8s.AndNot, andNotSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.AndNot, andNotSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.AndNot, andNotSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.AndNot, andNotSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.AndNot, andNotSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.AndNot, andNotSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.AndNot, andNotSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.AndNot, andNotSlice[uint64]) } func TestMulSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - mulFloat64 := func(x, y []float64) []float64 { - r := make([]float64, len(x)) - for i := range x { - r[i] = x[i] * y[i] - } - return r - } - testFloat64sBinary(t, archsimd.Float64s.Mul, mulFloat64) - mulInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = x[i] * y[i] - } - return r - } - // Ungated: compiles to the merging-predicated fallback, correct on any SVE. - testInt8sBinary(t, archsimd.Int8s.Mul, mulInt8) + testFloat32sBinary(t, archsimd.Float32s.Mul, mulSlice[float32]) + testFloat64sBinary(t, archsimd.Float64s.Mul, mulSlice[float64]) + // Ungated: integer Mul compiles to the merging-predicated fallback, + // correct on any SVE. + testInt8sBinary(t, archsimd.Int8s.Mul, mulSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.Mul, mulSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.Mul, mulSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.Mul, mulSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.Mul, mulSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Mul, mulSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Mul, mulSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Mul, mulSlice[uint64]) if archsimd.ARM64.SVE2() { // Gated: this block compiles to the unpredicated SVE2 encoding. - testInt8sBinary(t, archsimd.Int8s.Mul, mulInt8) + testInt8sBinary(t, archsimd.Int8s.Mul, mulSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.Mul, mulSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.Mul, mulSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.Mul, mulSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.Mul, mulSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.Mul, mulSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.Mul, mulSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.Mul, mulSlice[uint64]) } } @@ -367,26 +344,24 @@ func TestMulHighSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no SVE") } - mulHighInt8 := func(x, y []int8) []int8 { - r := make([]int8, len(x)) - for i := range x { - r[i] = int8((int16(x[i]) * int16(y[i])) >> 8) - } - return r - } - mulHighUint8 := func(x, y []uint8) []uint8 { - r := make([]uint8, len(x)) - for i := range x { - r[i] = uint8((uint16(x[i]) * uint16(y[i])) >> 8) - } - return r - } // Ungated: compiles to the merging-predicated fallback, correct on any SVE. - testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighInt8) - testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighUint8) + testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.MulHigh, mulHighSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.MulHigh, mulHighSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.MulHigh, mulHighSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.MulHigh, mulHighSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.MulHigh, mulHighSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.MulHigh, mulHighSlice[uint64]) if archsimd.ARM64.SVE2() { // Gated: this block compiles to the unpredicated SVE2 encodings. - testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighInt8) - testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighUint8) + testInt8sBinary(t, archsimd.Int8s.MulHigh, mulHighSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.MulHigh, mulHighSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.MulHigh, mulHighSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.MulHigh, mulHighSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.MulHigh, mulHighSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.MulHigh, mulHighSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.MulHigh, mulHighSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.MulHigh, mulHighSlice[uint64]) } } diff --git a/src/simd/archsimd/internal/simd_test/compare_sve_arm64_test.go b/src/simd/archsimd/internal/simd_test/compare_sve_arm64_test.go index 73c4f8561c77ba..1a1986b426bdc9 100644 --- a/src/simd/archsimd/internal/simd_test/compare_sve_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/compare_sve_arm64_test.go @@ -74,17 +74,40 @@ func testFloat64sCompare(t *testing.T, cmp func(_, _ archsimd.Float64s) archsimd testSVECompare(t, float64s, 8, z.Len(), archsimd.LoadFloat64s, cmp, archsimd.Mask64s.Store, want) } +func testUint16sCompare(t *testing.T, cmp func(_, _ archsimd.Uint16s) archsimd.Mask16s, want func(_, _ uint16) bool) { + var z archsimd.Uint16s + testSVECompare(t, uint16s, 2, z.Len(), archsimd.LoadUint16s, cmp, archsimd.Mask16s.Store, want) +} + +func testUint32sCompare(t *testing.T, cmp func(_, _ archsimd.Uint32s) archsimd.Mask32s, want func(_, _ uint32) bool) { + var z archsimd.Uint32s + testSVECompare(t, uint32s, 4, z.Len(), archsimd.LoadUint32s, cmp, archsimd.Mask32s.Store, want) +} + +func testUint64sCompare(t *testing.T, cmp func(_, _ archsimd.Uint64s) archsimd.Mask64s, want func(_, _ uint64) bool) { + var z archsimd.Uint64s + testSVECompare(t, uint64s, 8, z.Len(), archsimd.LoadUint64s, cmp, archsimd.Mask64s.Store, want) +} + +func gtWant[T number](a, b T) bool { return a > b } +func geWant[T number](a, b T) bool { return a >= b } +func eqWant[T number](a, b T) bool { return a == b } +func neWant[T number](a, b T) bool { return a != b } + func TestGreaterSVE(t *testing.T) { if !archsimd.ARM64.SVE() { - t.Skip("no sve") + t.Skip("no SVE") } - testInt8sCompare(t, archsimd.Int8s.Greater, func(a, b int8) bool { return a > b }) - testInt16sCompare(t, archsimd.Int16s.Greater, func(a, b int16) bool { return a > b }) - testInt32sCompare(t, archsimd.Int32s.Greater, func(a, b int32) bool { return a > b }) - testInt64sCompare(t, archsimd.Int64s.Greater, func(a, b int64) bool { return a > b }) - testUint8sCompare(t, archsimd.Uint8s.Greater, func(a, b uint8) bool { return a > b }) - testFloat32sCompare(t, archsimd.Float32s.Greater, func(a, b float32) bool { return a > b }) - testFloat64sCompare(t, archsimd.Float64s.Greater, func(a, b float64) bool { return a > b }) + testInt8sCompare(t, archsimd.Int8s.Greater, gtWant[int8]) + testInt16sCompare(t, archsimd.Int16s.Greater, gtWant[int16]) + testInt32sCompare(t, archsimd.Int32s.Greater, gtWant[int32]) + testInt64sCompare(t, archsimd.Int64s.Greater, gtWant[int64]) + testUint8sCompare(t, archsimd.Uint8s.Greater, gtWant[uint8]) + testUint16sCompare(t, archsimd.Uint16s.Greater, gtWant[uint16]) + testUint32sCompare(t, archsimd.Uint32s.Greater, gtWant[uint32]) + testUint64sCompare(t, archsimd.Uint64s.Greater, gtWant[uint64]) + testFloat32sCompare(t, archsimd.Float32s.Greater, gtWant[float32]) + testFloat64sCompare(t, archsimd.Float64s.Greater, gtWant[float64]) } // TestMaskStoreLoadPanicSVE checks that the exported mask memory APIs panic when @@ -105,27 +128,48 @@ func TestMaskStoreLoadPanicSVE(t *testing.T) { func TestEqualSVE(t *testing.T) { if !archsimd.ARM64.SVE() { - t.Skip("no sve") + t.Skip("no SVE") } - testInt8sCompare(t, archsimd.Int8s.Equal, func(a, b int8) bool { return a == b }) - testUint8sCompare(t, archsimd.Uint8s.Equal, func(a, b uint8) bool { return a == b }) - testFloat64sCompare(t, archsimd.Float64s.Equal, func(a, b float64) bool { return a == b }) + testInt8sCompare(t, archsimd.Int8s.Equal, eqWant[int8]) + testInt16sCompare(t, archsimd.Int16s.Equal, eqWant[int16]) + testInt32sCompare(t, archsimd.Int32s.Equal, eqWant[int32]) + testInt64sCompare(t, archsimd.Int64s.Equal, eqWant[int64]) + testUint8sCompare(t, archsimd.Uint8s.Equal, eqWant[uint8]) + testUint16sCompare(t, archsimd.Uint16s.Equal, eqWant[uint16]) + testUint32sCompare(t, archsimd.Uint32s.Equal, eqWant[uint32]) + testUint64sCompare(t, archsimd.Uint64s.Equal, eqWant[uint64]) + testFloat32sCompare(t, archsimd.Float32s.Equal, eqWant[float32]) + testFloat64sCompare(t, archsimd.Float64s.Equal, eqWant[float64]) } func TestNotEqualSVE(t *testing.T) { if !archsimd.ARM64.SVE() { - t.Skip("no sve") + t.Skip("no SVE") } - testInt8sCompare(t, archsimd.Int8s.NotEqual, func(a, b int8) bool { return a != b }) - testUint8sCompare(t, archsimd.Uint8s.NotEqual, func(a, b uint8) bool { return a != b }) - testFloat64sCompare(t, archsimd.Float64s.NotEqual, func(a, b float64) bool { return a != b }) + testInt8sCompare(t, archsimd.Int8s.NotEqual, neWant[int8]) + testInt16sCompare(t, archsimd.Int16s.NotEqual, neWant[int16]) + testInt32sCompare(t, archsimd.Int32s.NotEqual, neWant[int32]) + testInt64sCompare(t, archsimd.Int64s.NotEqual, neWant[int64]) + testUint8sCompare(t, archsimd.Uint8s.NotEqual, neWant[uint8]) + testUint16sCompare(t, archsimd.Uint16s.NotEqual, neWant[uint16]) + testUint32sCompare(t, archsimd.Uint32s.NotEqual, neWant[uint32]) + testUint64sCompare(t, archsimd.Uint64s.NotEqual, neWant[uint64]) + testFloat32sCompare(t, archsimd.Float32s.NotEqual, neWant[float32]) + testFloat64sCompare(t, archsimd.Float64s.NotEqual, neWant[float64]) } func TestGreaterEqualSVE(t *testing.T) { if !archsimd.ARM64.SVE() { - t.Skip("no sve") + t.Skip("no SVE") } - testInt8sCompare(t, archsimd.Int8s.GreaterEqual, func(a, b int8) bool { return a >= b }) - testUint8sCompare(t, archsimd.Uint8s.GreaterEqual, func(a, b uint8) bool { return a >= b }) - testFloat64sCompare(t, archsimd.Float64s.GreaterEqual, func(a, b float64) bool { return a >= b }) + testInt8sCompare(t, archsimd.Int8s.GreaterEqual, geWant[int8]) + testInt16sCompare(t, archsimd.Int16s.GreaterEqual, geWant[int16]) + testInt32sCompare(t, archsimd.Int32s.GreaterEqual, geWant[int32]) + testInt64sCompare(t, archsimd.Int64s.GreaterEqual, geWant[int64]) + testUint8sCompare(t, archsimd.Uint8s.GreaterEqual, geWant[uint8]) + testUint16sCompare(t, archsimd.Uint16s.GreaterEqual, geWant[uint16]) + testUint32sCompare(t, archsimd.Uint32s.GreaterEqual, geWant[uint32]) + testUint64sCompare(t, archsimd.Uint64s.GreaterEqual, geWant[uint64]) + testFloat32sCompare(t, archsimd.Float32s.GreaterEqual, geWant[float32]) + testFloat64sCompare(t, archsimd.Float64s.GreaterEqual, geWant[float64]) } diff --git a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go index 420bd7cd423741..c107901272fa3c 100644 --- a/src/simd/archsimd/internal/simd_test/simd_arm64_test.go +++ b/src/simd/archsimd/internal/simd_test/simd_arm64_test.go @@ -127,7 +127,8 @@ func TestAddSVESpill(t *testing.T) { } } -// TestAddSaturatedSVE checks that the generated saturating add saturates. +// TestAddSaturatedSVE checks the generated saturating add: an explicit +// boundary case, then every integer shape against the generic emulation. func TestAddSaturatedSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no sve") @@ -143,20 +144,18 @@ func TestAddSaturatedSVE(t *testing.T) { t.Errorf("int8 lane %d: got %d, want 127", i, gi[i]) } } - var su, gu [32]uint8 - for i := range su { - su[i] = 200 // 200+200 saturates to 255 - } - vu := archsimd.LoadUint8s(su[:]) - vu.AddSaturated(vu).Store(gu[:]) - for i := 0; i < vu.Len(); i++ { - if gu[i] != 255 { - t.Errorf("uint8 lane %d: got %d, want 255", i, gu[i]) - } - } + testInt8sBinary(t, archsimd.Int8s.AddSaturated, addSaturatedSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.AddSaturated, addSaturatedSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.AddSaturated, addSaturatedSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.AddSaturated, addSaturatedSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.AddSaturated, addSaturatedSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.AddSaturated, addSaturatedSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.AddSaturated, addSaturatedSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.AddSaturated, addSaturatedSlice[uint64]) } -// TestSubSaturatedSVE checks that the generated saturating subtract saturates. +// TestSubSaturatedSVE checks the generated saturating subtract: an explicit +// boundary case, then every integer shape against the generic emulation. func TestSubSaturatedSVE(t *testing.T) { if !archsimd.ARM64.SVE() { t.Skip("no sve") @@ -173,18 +172,14 @@ func TestSubSaturatedSVE(t *testing.T) { t.Errorf("int8 lane %d: got %d, want 127", i, gi[i]) } } - var ux, uy, gu [32]uint8 - for i := range ux { - ux[i] = 10 // 10 - 20 saturates to 0 - uy[i] = 20 - } - vx, vy := archsimd.LoadUint8s(ux[:]), archsimd.LoadUint8s(uy[:]) - vx.SubSaturated(vy).Store(gu[:]) - for i := 0; i < vx.Len(); i++ { - if gu[i] != 0 { - t.Errorf("uint8 lane %d: got %d, want 0", i, gu[i]) - } - } + testInt8sBinary(t, archsimd.Int8s.SubSaturated, subSaturatedSlice[int8]) + testInt16sBinary(t, archsimd.Int16s.SubSaturated, subSaturatedSlice[int16]) + testInt32sBinary(t, archsimd.Int32s.SubSaturated, subSaturatedSlice[int32]) + testInt64sBinary(t, archsimd.Int64s.SubSaturated, subSaturatedSlice[int64]) + testUint8sBinary(t, archsimd.Uint8s.SubSaturated, subSaturatedSlice[uint8]) + testUint16sBinary(t, archsimd.Uint16s.SubSaturated, subSaturatedSlice[uint16]) + testUint32sBinary(t, archsimd.Uint32s.SubSaturated, subSaturatedSlice[uint32]) + testUint64sBinary(t, archsimd.Uint64s.SubSaturated, subSaturatedSlice[uint64]) } func TestStringSVE(t *testing.T) { diff --git a/src/simd/archsimd/internal/simd_test/simulation_helpers_test.go b/src/simd/archsimd/internal/simd_test/simulation_helpers_test.go index 19390a9af4c453..1264c4d9732e76 100644 --- a/src/simd/archsimd/internal/simd_test/simulation_helpers_test.go +++ b/src/simd/archsimd/internal/simd_test/simulation_helpers_test.go @@ -385,6 +385,121 @@ func roundSlice[T float](x []T) []T { return map1[T](round)(x) } +func negSlice[T number](x []T) []T { + return map1[T](neg)(x) +} + +func absSlice[T number](x []T) []T { + return map1[T](abs)(x) +} + +// isSignedInt reports whether T is a signed integer type. +func isSignedInt[T integer]() bool { + var t T + return t-1 < 0 +} + +// addSaturated adds with saturation to T's range. Lanes narrower than 64 bits +// compute in 64-bit and clamp; 64-bit lanes detect overflow directly. +func addSaturated[T integer](a, b T) T { + size := uint(unsafe.Sizeof(a)) * 8 + if size < 64 { + if isSignedInt[T]() { + s := int64(a) + int64(b) + hi := int64(1)<<(size-1) - 1 + lo := -hi - 1 + return T(min(max(s, lo), hi)) + } + s := uint64(a) + uint64(b) + return T(min(s, uint64(1)< 0 && s < a { + return T(maxI) + } + if b < 0 && s > a { + return T(minI) + } + return s + } + if s < a { + return ^T(0) + } + return s +} + +// subSaturated subtracts with saturation to T's range. +func subSaturated[T integer](a, b T) T { + size := uint(unsafe.Sizeof(a)) * 8 + if size < 64 { + if isSignedInt[T]() { + s := int64(a) - int64(b) + hi := int64(1)<<(size-1) - 1 + lo := -hi - 1 + return T(min(max(s, lo), hi)) + } + if b > a { + return 0 + } + return a - b + } + s := a - b + if isSignedInt[T]() { + maxI, minI := int64(math.MaxInt64), int64(math.MinInt64) + if b < 0 && s < a { + return T(maxI) + } + if b > 0 && s > a { + return T(minI) + } + return s + } + if b > a { + return 0 + } + return s +} + +// mulHigh returns the high half of the full-width product. Lanes narrower than +// 64 bits widen; 64-bit lanes use bits.Mul64, adjusting for signedness. +func mulHigh[T integer](a, b T) T { + size := uint(unsafe.Sizeof(a)) * 8 + if size < 64 { + if isSignedInt[T]() { + return T((int64(a) * int64(b)) >> size) + } + return T((uint64(a) * uint64(b)) >> size) + } + hi, _ := bits.Mul64(uint64(a), uint64(b)) + if isSignedInt[T]() { + shi := int64(hi) + if a < 0 { + shi -= int64(b) + } + if b < 0 { + shi -= int64(a) + } + return T(shi) + } + return T(hi) +} + +func addSaturatedSlice[T integer](x, y []T) []T { + return map2[T](addSaturated)(x, y) +} + +func subSaturatedSlice[T integer](x, y []T) []T { + return map2[T](subSaturated)(x, y) +} + +func mulHighSlice[T integer](x, y []T) []T { + return map2[T](mulHigh)(x, y) +} + // lanewiseSlice is the common helper for interleave, deinterleave, and transpose // simulations. It handles lane computation, allocation, and iteration. // laneBits is the lane size in bits (128 for NEON/x86 128-bit, 0 for whole-input/SVE).