diff --git a/src/cmd/compile/internal/amd64/simdssa.go b/src/cmd/compile/internal/amd64/simdssa.go index 7c6872f469a27e..0241aae938710d 100644 --- a/src/cmd/compile/internal/amd64/simdssa.go +++ b/src/cmd/compile/internal/amd64/simdssa.go @@ -290,6 +290,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPHADDD256, ssaop.OpAMD64VPHADDSW128, ssaop.OpAMD64VPHADDSW256, + ssaop.OpAMD64VPACKSSDW128, + ssaop.OpAMD64VPACKSSDW256, + ssaop.OpAMD64VPACKSSDW512, + ssaop.OpAMD64VPACKUSDW128, + ssaop.OpAMD64VPACKUSDW256, + ssaop.OpAMD64VPACKUSDW512, ssaop.OpAMD64VHSUBPS128, ssaop.OpAMD64VHSUBPD128, ssaop.OpAMD64VPHSUBW128, @@ -479,12 +485,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPRORVQ128, ssaop.OpAMD64VPRORVQ256, ssaop.OpAMD64VPRORVQ512, - ssaop.OpAMD64VPACKSSDW128, - ssaop.OpAMD64VPACKSSDW256, - ssaop.OpAMD64VPACKSSDW512, - ssaop.OpAMD64VPACKUSDW128, - ssaop.OpAMD64VPACKUSDW256, - ssaop.OpAMD64VPACKUSDW512, ssaop.OpAMD64VSCALEFPS128, ssaop.OpAMD64VSCALEFPS256, ssaop.OpAMD64VSCALEFPS512, @@ -615,6 +615,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPAVGWMasked128, ssaop.OpAMD64VPAVGWMasked256, ssaop.OpAMD64VPAVGWMasked512, + ssaop.OpAMD64VPACKSSDWMasked256, + ssaop.OpAMD64VPACKSSDWMasked512, + ssaop.OpAMD64VPACKSSDWMasked128, + ssaop.OpAMD64VPACKUSDWMasked256, + ssaop.OpAMD64VPACKUSDWMasked512, + ssaop.OpAMD64VPACKUSDWMasked128, ssaop.OpAMD64VDIVPSMasked128, ssaop.OpAMD64VDIVPSMasked256, ssaop.OpAMD64VDIVPSMasked512, @@ -746,12 +752,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPRORVQMasked128, ssaop.OpAMD64VPRORVQMasked256, ssaop.OpAMD64VPRORVQMasked512, - ssaop.OpAMD64VPACKSSDWMasked256, - ssaop.OpAMD64VPACKSSDWMasked512, - ssaop.OpAMD64VPACKSSDWMasked128, - ssaop.OpAMD64VPACKUSDWMasked256, - ssaop.OpAMD64VPACKUSDWMasked512, - ssaop.OpAMD64VPACKUSDWMasked128, ssaop.OpAMD64VSCALEFPSMasked128, ssaop.OpAMD64VSCALEFPSMasked256, ssaop.OpAMD64VSCALEFPSMasked512, @@ -1432,6 +1432,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPERMI2QMasked256, ssaop.OpAMD64VPERMI2PDMasked512, ssaop.OpAMD64VPERMI2QMasked512, + ssaop.OpAMD64VPACKSSDWMasked256Merging, + ssaop.OpAMD64VPACKSSDWMasked512Merging, + ssaop.OpAMD64VPACKSSDWMasked128Merging, + ssaop.OpAMD64VPACKUSDWMasked256Merging, + ssaop.OpAMD64VPACKUSDWMasked512Merging, + ssaop.OpAMD64VPACKUSDWMasked128Merging, ssaop.OpAMD64VDIVPSMasked128Merging, ssaop.OpAMD64VDIVPSMasked256Merging, ssaop.OpAMD64VDIVPSMasked512Merging, @@ -1567,12 +1573,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPRORVQMasked128Merging, ssaop.OpAMD64VPRORVQMasked256Merging, ssaop.OpAMD64VPRORVQMasked512Merging, - ssaop.OpAMD64VPACKSSDWMasked256Merging, - ssaop.OpAMD64VPACKSSDWMasked512Merging, - ssaop.OpAMD64VPACKSSDWMasked128Merging, - ssaop.OpAMD64VPACKUSDWMasked256Merging, - ssaop.OpAMD64VPACKUSDWMasked512Merging, - ssaop.OpAMD64VPACKUSDWMasked128Merging, ssaop.OpAMD64VSCALEFPSMasked128Merging, ssaop.OpAMD64VSCALEFPSMasked256Merging, ssaop.OpAMD64VSCALEFPSMasked512Merging, @@ -1974,6 +1974,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPHADDD256load, ssaop.OpAMD64VPHADDSW128load, ssaop.OpAMD64VPHADDSW256load, + ssaop.OpAMD64VPACKSSDW128load, + ssaop.OpAMD64VPACKSSDW256load, + ssaop.OpAMD64VPACKSSDW512load, + ssaop.OpAMD64VPACKUSDW128load, + ssaop.OpAMD64VPACKUSDW256load, + ssaop.OpAMD64VPACKUSDW512load, ssaop.OpAMD64VHSUBPS128load, ssaop.OpAMD64VHSUBPD128load, ssaop.OpAMD64VPHSUBW128load, @@ -2142,12 +2148,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPRORVQ128load, ssaop.OpAMD64VPRORVQ256load, ssaop.OpAMD64VPRORVQ512load, - ssaop.OpAMD64VPACKSSDW128load, - ssaop.OpAMD64VPACKSSDW256load, - ssaop.OpAMD64VPACKSSDW512load, - ssaop.OpAMD64VPACKUSDW128load, - ssaop.OpAMD64VPACKUSDW256load, - ssaop.OpAMD64VPACKUSDW512load, ssaop.OpAMD64VSCALEFPS128load, ssaop.OpAMD64VSCALEFPS256load, ssaop.OpAMD64VSCALEFPS512load, @@ -2360,6 +2360,12 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPAVGBMasked256load, ssaop.OpAMD64VPAVGWMasked128load, ssaop.OpAMD64VPAVGWMasked256load, + ssaop.OpAMD64VPACKSSDWMasked256load, + ssaop.OpAMD64VPACKSSDWMasked512load, + ssaop.OpAMD64VPACKSSDWMasked128load, + ssaop.OpAMD64VPACKUSDWMasked256load, + ssaop.OpAMD64VPACKUSDWMasked512load, + ssaop.OpAMD64VPACKUSDWMasked128load, ssaop.OpAMD64VDIVPSMasked128load, ssaop.OpAMD64VDIVPSMasked256load, ssaop.OpAMD64VDIVPSMasked512load, @@ -2474,12 +2480,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPRORVQMasked128load, ssaop.OpAMD64VPRORVQMasked256load, ssaop.OpAMD64VPRORVQMasked512load, - ssaop.OpAMD64VPACKSSDWMasked256load, - ssaop.OpAMD64VPACKSSDWMasked512load, - ssaop.OpAMD64VPACKSSDWMasked128load, - ssaop.OpAMD64VPACKUSDWMasked256load, - ssaop.OpAMD64VPACKUSDWMasked512load, - ssaop.OpAMD64VPACKUSDWMasked128load, ssaop.OpAMD64VSCALEFPSMasked128load, ssaop.OpAMD64VSCALEFPSMasked256load, ssaop.OpAMD64VSCALEFPSMasked512load, @@ -3401,6 +3401,18 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPERMI2PDMasked512load, ssaop.OpAMD64VPERMI2QMasked512, ssaop.OpAMD64VPERMI2QMasked512load, + ssaop.OpAMD64VPACKSSDWMasked256, + ssaop.OpAMD64VPACKSSDWMasked256load, + ssaop.OpAMD64VPACKSSDWMasked512, + ssaop.OpAMD64VPACKSSDWMasked512load, + ssaop.OpAMD64VPACKSSDWMasked128, + ssaop.OpAMD64VPACKSSDWMasked128load, + ssaop.OpAMD64VPACKUSDWMasked256, + ssaop.OpAMD64VPACKUSDWMasked256load, + ssaop.OpAMD64VPACKUSDWMasked512, + ssaop.OpAMD64VPACKUSDWMasked512load, + ssaop.OpAMD64VPACKUSDWMasked128, + ssaop.OpAMD64VPACKUSDWMasked128load, ssaop.OpAMD64VPALIGNRMasked256, ssaop.OpAMD64VPALIGNRMasked256load, ssaop.OpAMD64VPALIGNRMasked512, @@ -3964,12 +3976,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPMOVSQBMasked128_128, ssaop.OpAMD64VPMOVSQBMasked128_256, ssaop.OpAMD64VPMOVSQBMasked128_512, - ssaop.OpAMD64VPACKSSDWMasked256, - ssaop.OpAMD64VPACKSSDWMasked256load, - ssaop.OpAMD64VPACKSSDWMasked512, - ssaop.OpAMD64VPACKSSDWMasked512load, - ssaop.OpAMD64VPACKSSDWMasked128, - ssaop.OpAMD64VPACKSSDWMasked128load, ssaop.OpAMD64VPMOVSDWMasked128_128, ssaop.OpAMD64VPMOVSDWMasked128_256, ssaop.OpAMD64VPMOVSDWMasked256, @@ -3988,12 +3994,6 @@ func ssaGenSIMDValue(s *ssagen.State, v *ssa.Value) bool { ssaop.OpAMD64VPMOVUSQBMasked128_128, ssaop.OpAMD64VPMOVUSQBMasked128_256, ssaop.OpAMD64VPMOVUSQBMasked128_512, - ssaop.OpAMD64VPACKUSDWMasked256, - ssaop.OpAMD64VPACKUSDWMasked256load, - ssaop.OpAMD64VPACKUSDWMasked512, - ssaop.OpAMD64VPACKUSDWMasked512load, - ssaop.OpAMD64VPACKUSDWMasked128, - ssaop.OpAMD64VPACKUSDWMasked128load, ssaop.OpAMD64VPMOVUSDWMasked128_128, ssaop.OpAMD64VPMOVUSDWMasked128_256, ssaop.OpAMD64VPMOVUSDWMasked256, diff --git a/src/cmd/compile/internal/midway/deepcopy.go b/src/cmd/compile/internal/midway/deepcopy.go index 99f6ef0a6f471f..b6590f87cefa2b 100644 --- a/src/cmd/compile/internal/midway/deepcopy.go +++ b/src/cmd/compile/internal/midway/deepcopy.go @@ -516,13 +516,13 @@ func (c *DeepCopier) CopySimpleStmt(s syntax.SimpleStmt) syntax.SimpleStmt { newList.ElemList = append(newList.ElemList, c.CopyExpr(el)) } } + newList.SetPos(list.Pos()) newS.Lhs = newList } else if id, ok := s.Lhs.(*syntax.Name); ok && s.Def { newS.Lhs = c.CopyName(id, true) } else { newS.Lhs = c.CopyExpr(s.Lhs) } - newS.Lhs.SetPos(s.Lhs.Pos()) newS.SetPos(s.Pos()) return newS case *syntax.AssignStmt: diff --git a/src/cmd/compile/internal/riscv64/ssa.go b/src/cmd/compile/internal/riscv64/ssa.go index a0fc15f73cebed..bd9e10fb1dde1e 100644 --- a/src/cmd/compile/internal/riscv64/ssa.go +++ b/src/cmd/compile/internal/riscv64/ssa.go @@ -784,10 +784,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { case ssaop.OpRISCV64LoweredZero: ptr := v.Args[0].Reg() - sc := v.AuxValAndOff() - n := sc.Val64() - - mov, sz := largestMove(sc.Off64()) + n, align := v.AuxSizeAndAlign() + mov, sz := largestMove(align) // mov ZERO, (offset)(Rarg0) var off int64 @@ -809,9 +807,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { case ssaop.OpRISCV64LoweredZeroLoop: ptr := v.Args[0].Reg() - sc := v.AuxValAndOff() - n := sc.Val64() - mov, sz := largestMove(sc.Off64()) + n, align := v.AuxSizeAndAlign() + mov, sz := largestMove(align) chunk := 8 * sz if n <= 3*chunk { @@ -820,9 +817,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { tmp := v.RegTmp() + if n >= 1<<31 { + p := s.Prog(riscv.AMOV) + p.From.Type = obj.TYPE_CONST + p.From.Offset = n - n%chunk + p.To.Type = obj.TYPE_REG + p.To.Reg = tmp + } p := s.Prog(riscv.AADD) - p.From.Type = obj.TYPE_CONST - p.From.Offset = n - n%chunk + if n >= 1<<31 { + p.From.Type = obj.TYPE_REG + p.From.Reg = tmp + } else { + p.From.Type = obj.TYPE_CONST + p.From.Offset = n - n%chunk + } p.Reg = ptr p.To.Type = obj.TYPE_REG p.To.Reg = tmp @@ -871,9 +880,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { break } - sa := v.AuxValAndOff() - n := sa.Val64() - mov, sz := largestMove(sa.Off64()) + n, align := v.AuxSizeAndAlign() + mov, sz := largestMove(align) var off int64 tmp := int16(riscv.REG_X5) @@ -900,9 +908,8 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { break } - sc := v.AuxValAndOff() - n := sc.Val64() - mov, sz := largestMove(sc.Off64()) + n, align := v.AuxSizeAndAlign() + mov, sz := largestMove(align) chunk := 8 * sz if n <= 3*chunk { @@ -910,9 +917,21 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) { } tmp := int16(riscv.REG_X5) + if n >= 1<<31 { + p := s.Prog(riscv.AMOV) + p.From.Type = obj.TYPE_CONST + p.From.Offset = n - n%chunk + p.To.Type = obj.TYPE_REG + p.To.Reg = riscv.REG_X6 + } p := s.Prog(riscv.AADD) - p.From.Type = obj.TYPE_CONST - p.From.Offset = n - n%chunk + if n >= 1<<31 { + p.From.Type = obj.TYPE_REG + p.From.Reg = riscv.REG_X6 + } else { + p.From.Type = obj.TYPE_CONST + p.From.Offset = n - n%chunk + } p.Reg = src p.To.Type = obj.TYPE_REG p.To.Reg = riscv.REG_X6 diff --git a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules index dca1bccde0b02c..425eb58797dbd9 100644 --- a/src/cmd/compile/internal/ssa/_gen/RISCV64.rules +++ b/src/cmd/compile/internal/ssa/_gen/RISCV64.rules @@ -379,11 +379,11 @@ // Unroll zeroing in medium size (at most 192 bytes i.e. 3 cachelines) (Zero [s] {t} ptr mem) && s <= 24*ssa.MoveSize(t.Alignment(), config) => - (LoweredZero [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] ptr mem) + (LoweredZero [s] {t.Alignment()} ptr mem) // Generic zeroing uses a loop (Zero [s] {t} ptr mem) && s > 24*ssa.MoveSize(t.Alignment(), config) => - (LoweredZeroLoop [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] ptr mem) + (LoweredZeroLoop [s] {t.Alignment()} ptr mem) // Checks (IsNonNil ...) => (SNEZ ...) @@ -449,12 +449,12 @@ // Generic move (Move [s] {t} dst src mem) && s > 0 && s <= 3*8*ssa.MoveSize(t.Alignment(), config) && ssa.LogLargeCopyValue(v, s) => - (LoweredMove [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] dst src mem) + (LoweredMove [s] {t.Alignment()} dst src mem) // Generic move uses a loop (Move [s] {t} dst src mem) && s > 3*8*ssa.MoveSize(t.Alignment(), config) && ssa.LogLargeCopyValue(v, s) => - (LoweredMoveLoop [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] dst src mem) + (LoweredMoveLoop [s] {t.Alignment()} dst src mem) // Boolean ops; 0=false, 1=true (AndB ...) => (AND ...) diff --git a/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go b/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go index 3fd3ee83fc13b0..6e00974f650dec 100644 --- a/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go +++ b/src/cmd/compile/internal/ssa/_gen/RISCV64Ops.go @@ -293,15 +293,15 @@ func init() { // general unrolled zeroing // arg0 = address of memory to zero // arg1 = mem - // auxint = element size and type alignment + // auxint = size + // aux = alignment (as an int64) // returns mem // mov ZERO, (OFFSET)(Rarg0) { name: "LoweredZero", - aux: "SymValAndOff", + aux: "SizeAndAlign", typ: "Mem", argLength: 2, - symEffect: "Write", faultOnNilArg0: true, addrSinkArg0: true, reg: regInfo{ @@ -310,15 +310,15 @@ func init() { }, // general unaligned zeroing // arg0 = address of memory to zero (clobber) - // arg2 = mem - // auxint = element size and type alignment + // arg1 = mem + // auxint = size + // aux = alignment (as an int64) // returns mem { name: "LoweredZeroLoop", - aux: "SymValAndOff", + aux: "SizeAndAlign", typ: "Mem", argLength: 2, - symEffect: "Write", needIntTemp: true, faultOnNilArg0: true, addrSinkArg0: true, @@ -332,14 +332,14 @@ func init() { // arg0 = address of dst memory (clobber) // arg1 = address of src memory (clobber) // arg2 = mem - // auxint = size and type alignment + // auxint = size + // aux = alignment (as an int64) // returns mem // mov (offset)(Rarg1), TMP // mov TMP, (offset)(Rarg0) { name: "LoweredMove", - aux: "SymValAndOff", - symEffect: "Write", + aux: "SizeAndAlign", argLength: 3, reg: regInfo{ inputs: []regMask{gpMask.minus(regNamed["X5"]), gpMask.minus(regNamed["X5"])}, @@ -354,8 +354,9 @@ func init() { // general unaligned move // arg0 = address of dst memory (clobber) // arg1 = address of src memory (clobber) - // arg3 = mem - // auxint = alignment + // arg2 = mem + // auxint = size + // aux = alignment (as an int64) // returns mem // ADD $sz, X6 //loop: @@ -367,9 +368,8 @@ func init() { // BNE X6, Rarg1, loop { name: "LoweredMoveLoop", - aux: "SymValAndOff", + aux: "SizeAndAlign", argLength: 3, - symEffect: "Write", reg: regInfo{ inputs: []regMask{gpMask.minus(r5toR6), gpMask.minus(r5toR6)}, clobbers: r5toR6, diff --git a/src/cmd/compile/internal/ssa/_gen/allocators.go b/src/cmd/compile/internal/ssa/_gen/allocators.go index 3fb49a31c6e967..9f8033f37a978a 100644 --- a/src/cmd/compile/internal/ssa/_gen/allocators.go +++ b/src/cmd/compile/internal/ssa/_gen/allocators.go @@ -67,7 +67,7 @@ func genAllocators() { }, { name: "SparseMap", - typ: "*sparseMap", + typ: "*SparseMap", capacity: "%s.cap()", mak: "newSparseMap(%s)", resize: "", // larger-sized sparse maps are ok diff --git a/src/cmd/compile/internal/ssa/_gen/main.go b/src/cmd/compile/internal/ssa/_gen/main.go index 47de313f60eef3..81113a26441b8e 100644 --- a/src/cmd/compile/internal/ssa/_gen/main.go +++ b/src/cmd/compile/internal/ssa/_gen/main.go @@ -513,13 +513,13 @@ func genOp() { } if v.faultOnNilArg0 { fmt.Fprintln(w, splitTitle("faultOnNilArg0: true,")) - if v.aux != "Sym" && v.aux != "SymOff" && v.aux != "SymValAndOff" && v.aux != "Int64" && v.aux != "Int32" && v.aux != "" { + if v.aux != "Sym" && v.aux != "SymOff" && v.aux != "SymValAndOff" && v.aux != "Int64" && v.aux != "Int32" && v.aux != "SizeAndAlign" && v.aux != "" { log.Fatalf("faultOnNilArg0 with aux %s not allowed", v.aux) } } if v.faultOnNilArg1 { fmt.Fprintln(w, splitTitle("faultOnNilArg1: true,")) - if v.aux != "Sym" && v.aux != "SymOff" && v.aux != "SymValAndOff" && v.aux != "Int64" && v.aux != "Int32" && v.aux != "" { + if v.aux != "Sym" && v.aux != "SymOff" && v.aux != "SymValAndOff" && v.aux != "Int64" && v.aux != "Int32" && v.aux != "SizeAndAlign" && v.aux != "" { log.Fatalf("faultOnNilArg1 with aux %s not allowed", v.aux) } } diff --git a/src/cmd/compile/internal/ssa/_gen/rulegen.go b/src/cmd/compile/internal/ssa/_gen/rulegen.go index d45b59a95c62f5..ee3fae37034f76 100644 --- a/src/cmd/compile/internal/ssa/_gen/rulegen.go +++ b/src/cmd/compile/internal/ssa/_gen/rulegen.go @@ -1489,7 +1489,7 @@ func opHasAuxInt(op opData) bool { switch op.aux { case "Bool", "Int8", "Int16", "Int32", "Int64", "Int128", "UInt8", "Float32", "Float64", "SymOff", "CallOff", "SymValAndOff", "TypSize", "ARM64BitField", "FlagConstant", "CCop", - "PanicBoundsC", "PanicBoundsCC", "ARM64ConditionalParams": + "PanicBoundsC", "PanicBoundsCC", "ARM64ConditionalParams", "SizeAndAlign": return true } return false @@ -1498,7 +1498,7 @@ func opHasAuxInt(op opData) bool { func opHasAux(op opData) bool { switch op.aux { case "String", "Sym", "SymOff", "Call", "CallOff", "SymValAndOff", "Typ", "TypSize", - "S390XCCMask", "S390XRotateParams", "PanicBoundsC", "PanicBoundsCC": + "S390XCCMask", "S390XRotateParams", "PanicBoundsC", "PanicBoundsCC", "SizeAndAlign": return true } return false @@ -1857,6 +1857,8 @@ func (op opData) auxType() string { return "PanicBoundsC" case "PanicBoundsCC": return "PanicBoundsCC" + case "SizeAndAlign": + return "int64" default: return "invalid" } @@ -1901,6 +1903,8 @@ func (op opData) auxIntType() string { return "arm64ConditionalParams" case "PanicBoundsC", "PanicBoundsCC": return "int64" + case "SizeAndAlign": + return "int64" default: return "invalid" } diff --git a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules index 694301009d284d..6bb3f28386942d 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules +++ b/src/cmd/compile/internal/ssa/_gen/simdAMD64.rules @@ -226,6 +226,12 @@ (ConcatPermute128ScalarsUint16x16 ...) => (VPERM2I128256 ...) // pureVreg (ConcatPermute128ScalarsUint32x8 ...) => (VPERM2I128256 ...) // pureVreg (ConcatPermute128ScalarsUint64x4 ...) => (VPERM2I128256 ...) // pureVreg +(ConcatSaturateToInt16Int32x4 ...) => (VPACKSSDW128 ...) // pureVreg +(ConcatSaturateToInt16GroupedInt32x8 ...) => (VPACKSSDW256 ...) // pureVreg +(ConcatSaturateToInt16GroupedInt32x16 ...) => (VPACKSSDW512 ...) // pureVreg +(ConcatSaturateToUint16Int32x4 ...) => (VPACKUSDW128 ...) // pureVreg +(ConcatSaturateToUint16GroupedInt32x8 ...) => (VPACKUSDW256 ...) // pureVreg +(ConcatSaturateToUint16GroupedInt32x16 ...) => (VPACKUSDW512 ...) // pureVreg (ConcatShiftBytesRightUint8x16 ...) => (VPALIGNR128 ...) // pureVreg (ConcatShiftBytesRightGroupedUint8x32 ...) => (VPALIGNR256 ...) // pureVreg (ConcatShiftBytesRightGroupedUint8x64 ...) => (VPALIGNR512 ...) // pureVreg @@ -883,9 +889,6 @@ (SaturateToInt16Int64x2 ...) => (VPMOVSQW128_128 ...) // pureVreg (SaturateToInt16Int64x4 ...) => (VPMOVSQW128_256 ...) // pureVreg (SaturateToInt16Int64x8 ...) => (VPMOVSQW128_512 ...) // pureVreg -(SaturateToInt16ConcatInt32x4 ...) => (VPACKSSDW128 ...) // pureVreg -(SaturateToInt16ConcatGroupedInt32x8 ...) => (VPACKSSDW256 ...) // pureVreg -(SaturateToInt16ConcatGroupedInt32x16 ...) => (VPACKSSDW512 ...) // pureVreg (SaturateToInt32Int64x2 ...) => (VPMOVSQD128_128 ...) // pureVreg (SaturateToInt32Int64x4 ...) => (VPMOVSQD128_256 ...) // pureVreg (SaturateToInt32Int64x8 ...) => (VPMOVSQD256 ...) // pureVreg @@ -904,9 +907,6 @@ (SaturateToUint16Uint64x2 ...) => (VPMOVUSQW128_128 ...) // pureVreg (SaturateToUint16Uint64x4 ...) => (VPMOVUSQW128_256 ...) // pureVreg (SaturateToUint16Uint64x8 ...) => (VPMOVUSQW128_512 ...) // pureVreg -(SaturateToUint16ConcatInt32x4 ...) => (VPACKUSDW128 ...) // pureVreg -(SaturateToUint16ConcatGroupedInt32x8 ...) => (VPACKUSDW256 ...) // pureVreg -(SaturateToUint16ConcatGroupedInt32x16 ...) => (VPACKUSDW512 ...) // pureVreg (SaturateToUint32Uint64x2 ...) => (VPMOVUSQD128_128 ...) // pureVreg (SaturateToUint32Uint64x4 ...) => (VPMOVUSQD128_256 ...) // pureVreg (SaturateToUint32Uint64x8 ...) => (VPMOVUSQD256 ...) // pureVreg @@ -1544,6 +1544,12 @@ (VMOVDQU64Masked256 (VPERMI2Q256 x y z) mask) => (VPERMI2QMasked256 x y z mask) (VMOVDQU64Masked512 (VPERMI2PD512 x y z) mask) => (VPERMI2PDMasked512 x y z mask) (VMOVDQU64Masked512 (VPERMI2Q512 x y z) mask) => (VPERMI2QMasked512 x y z mask) +(VMOVDQU32Masked256 (VPACKSSDW256 x y) mask) => (VPACKSSDWMasked256 x y mask) +(VMOVDQU32Masked512 (VPACKSSDW512 x y) mask) => (VPACKSSDWMasked512 x y mask) +(VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) => (VPACKSSDWMasked128 x y mask) +(VMOVDQU32Masked256 (VPACKUSDW256 x y) mask) => (VPACKUSDWMasked256 x y mask) +(VMOVDQU32Masked512 (VPACKUSDW512 x y) mask) => (VPACKUSDWMasked512 x y mask) +(VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) => (VPACKUSDWMasked128 x y mask) (VMOVDQU8Masked256 (VPALIGNR256 [a] x y) mask) => (VPALIGNRMasked256 [a] x y mask) (VMOVDQU8Masked512 (VPALIGNR512 [a] x y) mask) => (VPALIGNRMasked512 [a] x y mask) (VMOVDQU8Masked128 (VPALIGNR128 [a] x y) mask) => (VPALIGNRMasked128 [a] x y mask) @@ -1818,9 +1824,6 @@ (VMOVDQU64Masked128 (VPMOVSQB128_128 x) mask) => (VPMOVSQBMasked128_128 x mask) (VMOVDQU64Masked256 (VPMOVSQB128_256 x) mask) => (VPMOVSQBMasked128_256 x mask) (VMOVDQU64Masked512 (VPMOVSQB128_512 x) mask) => (VPMOVSQBMasked128_512 x mask) -(VMOVDQU32Masked256 (VPACKSSDW256 x y) mask) => (VPACKSSDWMasked256 x y mask) -(VMOVDQU32Masked512 (VPACKSSDW512 x y) mask) => (VPACKSSDWMasked512 x y mask) -(VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) => (VPACKSSDWMasked128 x y mask) (VMOVDQU32Masked128 (VPMOVSDW128_128 x) mask) => (VPMOVSDWMasked128_128 x mask) (VMOVDQU32Masked256 (VPMOVSDW128_256 x) mask) => (VPMOVSDWMasked128_256 x mask) (VMOVDQU32Masked256 (VPMOVSDW256 x) mask) => (VPMOVSDWMasked256 x mask) @@ -1839,9 +1842,6 @@ (VMOVDQU64Masked128 (VPMOVUSQB128_128 x) mask) => (VPMOVUSQBMasked128_128 x mask) (VMOVDQU64Masked256 (VPMOVUSQB128_256 x) mask) => (VPMOVUSQBMasked128_256 x mask) (VMOVDQU64Masked512 (VPMOVUSQB128_512 x) mask) => (VPMOVUSQBMasked128_512 x mask) -(VMOVDQU32Masked256 (VPACKUSDW256 x y) mask) => (VPACKUSDWMasked256 x y mask) -(VMOVDQU32Masked512 (VPACKUSDW512 x y) mask) => (VPACKUSDWMasked512 x y mask) -(VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) => (VPACKUSDWMasked128 x y mask) (VMOVDQU32Masked128 (VPMOVUSDW128_128 x) mask) => (VPMOVUSDWMasked128_128 x mask) (VMOVDQU32Masked256 (VPMOVUSDW128_256 x) mask) => (VPMOVUSDWMasked128_256 x mask) (VMOVDQU32Masked256 (VPMOVUSDW256 x) mask) => (VPMOVUSDWMasked256 x mask) @@ -2697,6 +2697,18 @@ (VPERMI2QMasked256 x y l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPERMI2QMasked256load {sym} [off] x y ptr mask mem) // vregMem (VPERMI2PDMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPERMI2PDMasked512load {sym} [off] x y ptr mask mem) // vregMem (VPERMI2QMasked512 x y l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPERMI2QMasked512load {sym} [off] x y ptr mask mem) // vregMem +(VPACKSSDW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW128load {sym} [off] x ptr mem) // vregMem +(VPACKSSDW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW256load {sym} [off] x ptr mem) // vregMem +(VPACKSSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW512load {sym} [off] x ptr mem) // vregMem +(VPACKSSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked256load {sym} [off] x ptr mask mem) // vregMem +(VPACKSSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked512load {sym} [off] x ptr mask mem) // vregMem +(VPACKSSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked128load {sym} [off] x ptr mask mem) // vregMem +(VPACKUSDW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW128load {sym} [off] x ptr mem) // vregMem +(VPACKUSDW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW256load {sym} [off] x ptr mem) // vregMem +(VPACKUSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW512load {sym} [off] x ptr mem) // vregMem +(VPACKUSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked256load {sym} [off] x ptr mask mem) // vregMem +(VPACKUSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked512load {sym} [off] x ptr mask mem) // vregMem +(VPACKUSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked128load {sym} [off] x ptr mask mem) // vregMem (VPALIGNR128 [c] x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPALIGNR128load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mem) // vregMem (VPALIGNR256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPALIGNR256load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mem) // vregMem (VPALIGNRMasked256 [c] x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPALIGNRMasked256load {sym} [ssa.MakeValAndOff(int32(uint8(c)),off)] x ptr mask mem) // vregMem @@ -3307,18 +3319,6 @@ (VPRORVQMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPRORVQMasked128load {sym} [off] x ptr mask mem) // vregMem (VPRORVQMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPRORVQMasked256load {sym} [off] x ptr mask mem) // vregMem (VPRORVQMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPRORVQMasked512load {sym} [off] x ptr mask mem) // vregMem -(VPACKSSDW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW128load {sym} [off] x ptr mem) // vregMem -(VPACKSSDW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW256load {sym} [off] x ptr mem) // vregMem -(VPACKSSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDW512load {sym} [off] x ptr mem) // vregMem -(VPACKSSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked256load {sym} [off] x ptr mask mem) // vregMem -(VPACKSSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked512load {sym} [off] x ptr mask mem) // vregMem -(VPACKSSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKSSDWMasked128load {sym} [off] x ptr mask mem) // vregMem -(VPACKUSDW128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW128load {sym} [off] x ptr mem) // vregMem -(VPACKUSDW256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW256load {sym} [off] x ptr mem) // vregMem -(VPACKUSDW512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDW512load {sym} [off] x ptr mem) // vregMem -(VPACKUSDWMasked256 x l:(VMOVDQUload256 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked256load {sym} [off] x ptr mask mem) // vregMem -(VPACKUSDWMasked512 x l:(VMOVDQUload512 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked512load {sym} [off] x ptr mask mem) // vregMem -(VPACKUSDWMasked128 x l:(VMOVDQUload128 {sym} [off] ptr mem) mask) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VPACKUSDWMasked128load {sym} [off] x ptr mask mem) // vregMem (VSCALEFPS128 x l:(VMOVDQUload128 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VSCALEFPS128load {sym} [off] x ptr mem) // vregMem (VSCALEFPS256 x l:(VMOVDQUload256 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VSCALEFPS256load {sym} [off] x ptr mem) // vregMem (VSCALEFPS512 x l:(VMOVDQUload512 {sym} [off] ptr mem)) && ssa.CanMergeLoad(v, l) && ssa.Clobber(l) => (VSCALEFPS512load {sym} [off] x ptr mem) // vregMem diff --git a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go index a82ea9546d1648..4052d5f02e3fdc 100644 --- a/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go +++ b/src/cmd/compile/internal/ssa/_gen/simdgenericOps.go @@ -267,6 +267,12 @@ func simdGenericOps() []opData { {name: "ConcatPermuteUint64x2", argLength: 3}, // ARCH:amd64 {name: "ConcatPermuteUint64x4", argLength: 3}, // ARCH:amd64 {name: "ConcatPermuteUint64x8", argLength: 3}, // ARCH:amd64 + {name: "ConcatSaturateToInt16GroupedInt32x8", argLength: 2}, // ARCH:amd64 + {name: "ConcatSaturateToInt16GroupedInt32x16", argLength: 2}, // ARCH:amd64 + {name: "ConcatSaturateToInt16Int32x4", argLength: 2}, // ARCH:amd64 + {name: "ConcatSaturateToUint16GroupedInt32x8", argLength: 2}, // ARCH:amd64 + {name: "ConcatSaturateToUint16GroupedInt32x16", argLength: 2}, // ARCH:amd64 + {name: "ConcatSaturateToUint16Int32x4", argLength: 2}, // ARCH:amd64 {name: "ConcatSubPairsFloat32x4", argLength: 2}, // ARCH:amd64 {name: "ConcatSubPairsFloat64x2", argLength: 2}, // ARCH:amd64 {name: "ConcatSubPairsGroupedFloat32x8", argLength: 2}, // ARCH:amd64 @@ -1007,9 +1013,6 @@ func simdGenericOps() []opData { {name: "SaturateToInt8Int64x2", argLength: 1}, // ARCH:amd64 {name: "SaturateToInt8Int64x4", argLength: 1}, // ARCH:amd64 {name: "SaturateToInt8Int64x8", argLength: 1}, // ARCH:amd64 - {name: "SaturateToInt16ConcatGroupedInt32x8", argLength: 2}, // ARCH:amd64 - {name: "SaturateToInt16ConcatGroupedInt32x16", argLength: 2}, // ARCH:amd64 - {name: "SaturateToInt16ConcatInt32x4", argLength: 2}, // ARCH:amd64 {name: "SaturateToInt16Int32x4", argLength: 1}, // ARCH:amd64,arm64 {name: "SaturateToInt16Int32x8", argLength: 1}, // ARCH:amd64 {name: "SaturateToInt16Int32x16", argLength: 1}, // ARCH:amd64 @@ -1029,9 +1032,6 @@ func simdGenericOps() []opData { {name: "SaturateToUint8Uint64x2", argLength: 1}, // ARCH:amd64 {name: "SaturateToUint8Uint64x4", argLength: 1}, // ARCH:amd64 {name: "SaturateToUint8Uint64x8", argLength: 1}, // ARCH:amd64 - {name: "SaturateToUint16ConcatGroupedInt32x8", argLength: 2}, // ARCH:amd64 - {name: "SaturateToUint16ConcatGroupedInt32x16", argLength: 2}, // ARCH:amd64 - {name: "SaturateToUint16ConcatInt32x4", argLength: 2}, // ARCH:amd64 {name: "SaturateToUint16Int32x4", argLength: 1}, // ARCH:arm64 {name: "SaturateToUint16Uint32x4", argLength: 1}, // ARCH:amd64,arm64 {name: "SaturateToUint16Uint32x8", argLength: 1}, // ARCH:amd64 diff --git a/src/cmd/compile/internal/ssa/allocators.go b/src/cmd/compile/internal/ssa/allocators.go index 3dbc1cf5bf87e1..f96620a9a04f64 100644 --- a/src/cmd/compile/internal/ssa/allocators.go +++ b/src/cmd/compile/internal/ssa/allocators.go @@ -106,8 +106,8 @@ func (c *Cache) FreeSparseSet(s *SparseSet) { var poolFreeSparseMap [27]sync.Pool -func (c *Cache) AllocSparseMap(n int) *sparseMap { - var s *sparseMap +func (c *Cache) AllocSparseMap(n int) *SparseMap { + var s *SparseMap n2 := n if n2 < 32 { n2 = 32 @@ -117,11 +117,11 @@ func (c *Cache) AllocSparseMap(n int) *sparseMap { if v == nil { s = NewSparseMap(1 << b) } else { - s = v.(*sparseMap) + s = v.(*SparseMap) } return s } -func (c *Cache) FreeSparseMap(s *sparseMap) { +func (c *Cache) FreeSparseMap(s *SparseMap) { s.Clear() b := bits.Len(uint(s.cap()) - 1) poolFreeSparseMap[b-5].Put(s) diff --git a/src/cmd/compile/internal/ssa/biasedsparsemap.go b/src/cmd/compile/internal/ssa/biasedsparsemap.go index 951e74eecdc310..1baa169f1d94fc 100644 --- a/src/cmd/compile/internal/ssa/biasedsparsemap.go +++ b/src/cmd/compile/internal/ssa/biasedsparsemap.go @@ -12,7 +12,7 @@ import "math" // Not all features of a SparseMap are exported, and it is also easy to treat a // BiasedSparseMap like a SparseSet. type BiasedSparseMap struct { - s *sparseMap + s *SparseMap first int } diff --git a/src/cmd/compile/internal/ssa/deadcode.go b/src/cmd/compile/internal/ssa/deadcode.go index d13e11a407ef30..84384af642beb8 100644 --- a/src/cmd/compile/internal/ssa/deadcode.go +++ b/src/cmd/compile/internal/ssa/deadcode.go @@ -8,6 +8,7 @@ import ( "cmd/compile/internal/ssa/block" "cmd/compile/internal/ssa/ssaop" "cmd/internal/src" + "math" ) // LiveValues returns the live values in f and a list of values that are eligible @@ -29,28 +30,15 @@ func LiveValues(f *Func, reachable []bool) (live []bool, liveOrderStmts []*Value return } - // Record all the inline indexes we need - var liveInlIdx map[int]bool - pt := f.Config.Ctxt.PosTable - for _, b := range f.Blocks { - for _, v := range b.Values { - i := pt.Pos(v.Pos).Base().InliningIndex() - if i < 0 { - continue - } - if liveInlIdx == nil { - liveInlIdx = map[int]bool{} - } - liveInlIdx[i] = true - } - i := pt.Pos(b.Pos).Base().InliningIndex() - if i < 0 { - continue - } - if liveInlIdx == nil { - liveInlIdx = map[int]bool{} - } - liveInlIdx[i] = true + // Record all the inline indexes we need. Most functions do not have any + // inlining, and if they do, the indexes are sparse so it makes sense to do + // probing before we allocate a set. + maxInl := collectInlineIdx(f, nil) + var liveInlIdx *SparseSet + if maxInl != -1 { + liveInlIdx = f.NewSparseSet(maxInl + 1) + defer f.RetSparseSet(liveInlIdx) + collectInlineIdx(f, liveInlIdx) } // Find all live values @@ -80,8 +68,9 @@ func LiveValues(f *Func, reachable []bool) (live []bool, liveOrderStmts []*Value liveOrderStmts = append(liveOrderStmts, v) } } + if v.Op == ssaop.OpInlMark { - if !liveInlIdx[int(v.AuxInt)] { + if liveInlIdx == nil || int(v.AuxInt) > maxInl || !liveInlIdx.Contains(ID(v.AuxInt)) { // We don't need marks for bodies that // have been completely optimized away. // TODO: save marks only for bodies which @@ -159,6 +148,36 @@ func findlive(f *Func) (reachable []bool, live []bool) { return } +func collectInlineIdx(f *Func, set *SparseSet) int { + pt := f.Config.Ctxt.PosTable + maxIndex := -1 + for _, b := range f.Blocks { + for _, v := range b.Values { + i := pt.Pos(v.Pos).Base().InliningIndex() + if i < 0 { + continue + } + maxIndex = max(maxIndex, i) + if set != nil { + set.Add(ID(i)) + } + } + i := pt.Pos(b.Pos).Base().InliningIndex() + if i < 0 { + continue + } + maxIndex = max(maxIndex, i) + if set != nil { + set.Add(ID(i)) + } + } + if maxIndex > math.MaxInt32 { + // punning between ID and int, make sure we fit + panic("very large inline index") + } + return maxIndex +} + // RemoveEdge removes the i'th outgoing edge from b (and // the corresponding incoming edge from b.Succs[i].b). // Note that this potentially reorders successors of b, so it diff --git a/src/cmd/compile/internal/ssa/func.go b/src/cmd/compile/internal/ssa/func.go index f88a7ce8e37709..800766ec9d9a5d 100644 --- a/src/cmd/compile/internal/ssa/func.go +++ b/src/cmd/compile/internal/ssa/func.go @@ -179,13 +179,13 @@ func (f *Func) RetSparseSet(ss *SparseSet) { } // NewSparseMap returns a sparse map that can store at least up to n integers. -func (f *Func) NewSparseMap(n int) *sparseMap { +func (f *Func) NewSparseMap(n int) *SparseMap { return f.Cache.AllocSparseMap(n) } // RetSparseMap returns a sparse map to the config's cache of sparse // sets to be reused by f.newSparseMap. -func (f *Func) RetSparseMap(ss *sparseMap) { +func (f *Func) RetSparseMap(ss *SparseMap) { f.Cache.FreeSparseMap(ss) } diff --git a/src/cmd/compile/internal/ssa/rewrite.go b/src/cmd/compile/internal/ssa/rewrite.go index ab1bd713cfdff0..746f9c9e82a1c4 100644 --- a/src/cmd/compile/internal/ssa/rewrite.go +++ b/src/cmd/compile/internal/ssa/rewrite.go @@ -527,6 +527,14 @@ func Arm64ConditionalParamsToAuxInt(v Arm64ConditionalParams) int64 { return i } +type Int64Aux int64 + +func (Int64Aux) CanBeAnSSAAux() {} + +func Int64ToAux(v int64) Aux { + return Int64Aux(v) +} + // encodes the lsb and width for arm(64) bitfield ops into the expected auxInt format. func ArmBFAuxInt(lsb, width int64) Arm64BitField { if lsb < 0 || lsb > 63 { diff --git a/src/cmd/compile/internal/ssa/sparsemap.go b/src/cmd/compile/internal/ssa/sparsemap.go index 93203115797a9e..1237144d45a72e 100644 --- a/src/cmd/compile/internal/ssa/sparsemap.go +++ b/src/cmd/compile/internal/ssa/sparsemap.go @@ -6,7 +6,7 @@ package ssa // NewSparseMap returns a sparseMap that can map // integers between 0 and n-1 to int32s. -func NewSparseMap(n int) *sparseMap { +func NewSparseMap(n int) *SparseMap { return newGenericSparseMap[ID, int32](n) } @@ -32,7 +32,7 @@ type sparseEntry[K sparseKey, V any] struct { // sparseKey needs to be something we can index a slice with. type sparseKey interface{ ~int | ~int32 } -type sparseMap = genericSparseMap[ID, int32] +type SparseMap = genericSparseMap[ID, int32] func (s *genericSparseMap[K, V]) cap() int { return len(s.sparse) diff --git a/src/cmd/compile/internal/ssa/ssaop/op.go b/src/cmd/compile/internal/ssa/ssaop/op.go index 56ea38fb9912b0..58f07aa223053f 100644 --- a/src/cmd/compile/internal/ssa/ssaop/op.go +++ b/src/cmd/compile/internal/ssa/ssaop/op.go @@ -46,6 +46,7 @@ const ( AuxTypeS390XCCMask // aux is a s390x 4-bit condition code mask AuxTypeS390XCCMaskInt8 // aux is a s390x 4-bit condition code mask, auxInt is an int8 immediate AuxTypeS390XCCMaskUint8 // aux is a s390x 4-bit condition code mask, auxInt is a uint8 immediate + AuxTypeSizeAndAlign // auxInt is an int64 size, aux is an int64 alignment ) // An Op encodes the specific operation that a Value performs. diff --git a/src/cmd/compile/internal/ssa/ssaop/opGen.go b/src/cmd/compile/internal/ssa/ssaop/opGen.go index 788e71c8d0022c..debf9d284726f2 100644 --- a/src/cmd/compile/internal/ssa/ssaop/opGen.go +++ b/src/cmd/compile/internal/ssa/ssaop/opGen.go @@ -7248,6 +7248,12 @@ const ( OpConcatPermuteUint8x16 OpConcatPermuteUint8x32 OpConcatPermuteUint8x64 + OpConcatSaturateToInt16GroupedInt32x16 + OpConcatSaturateToInt16GroupedInt32x8 + OpConcatSaturateToInt16Int32x4 + OpConcatSaturateToUint16GroupedInt32x16 + OpConcatSaturateToUint16GroupedInt32x8 + OpConcatSaturateToUint16Int32x4 OpConcatShiftBytesRightGroupedUint8x32 OpConcatShiftBytesRightGroupedUint8x64 OpConcatShiftBytesRightUint8x16 @@ -8023,9 +8029,6 @@ const ( OpSHA256Message1Uint32x4 OpSHA256Message2Uint32x4 OpSHA256TwoRoundsUint32x4 - OpSaturateToInt16ConcatGroupedInt32x16 - OpSaturateToInt16ConcatGroupedInt32x8 - OpSaturateToInt16ConcatInt32x4 OpSaturateToInt16Int32x16 OpSaturateToInt16Int32x4 OpSaturateToInt16Int32x8 @@ -8044,9 +8047,6 @@ const ( OpSaturateToInt8Int64x2 OpSaturateToInt8Int64x4 OpSaturateToInt8Int64x8 - OpSaturateToUint16ConcatGroupedInt32x16 - OpSaturateToUint16ConcatGroupedInt32x8 - OpSaturateToUint16ConcatInt32x4 OpSaturateToUint16Int32x4 OpSaturateToUint16Uint32x16 OpSaturateToUint16Uint32x4 @@ -97300,11 +97300,10 @@ var OpcodeTable = [...]OpInfo{ }, { Name: "LoweredZero", - AuxType: AuxTypeSymValAndOff, + AuxType: AuxTypeSizeAndAlign, ArgLen: 2, FaultOnNilArg0: true, AddrSinkArg0: true, - symEffect: SymWrite, Reg: RegInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 1006632944, V2: 0}}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30 @@ -97313,12 +97312,11 @@ var OpcodeTable = [...]OpInfo{ }, { Name: "LoweredZeroLoop", - AuxType: AuxTypeSymValAndOff, + AuxType: AuxTypeSizeAndAlign, ArgLen: 2, NeedIntTemp: true, FaultOnNilArg0: true, AddrSinkArg0: true, - symEffect: SymWrite, Reg: RegInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 1006632944, V2: 0}}, // X5 X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30 @@ -97328,13 +97326,12 @@ var OpcodeTable = [...]OpInfo{ }, { Name: "LoweredMove", - AuxType: AuxTypeSymValAndOff, + AuxType: AuxTypeSizeAndAlign, ArgLen: 3, FaultOnNilArg0: true, FaultOnNilArg1: true, AddrSinkArg0: true, AddrSinkArg1: true, - symEffect: SymWrite, Reg: RegInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 1006632928, V2: 0}}, // X6 X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30 @@ -97345,13 +97342,12 @@ var OpcodeTable = [...]OpInfo{ }, { Name: "LoweredMoveLoop", - AuxType: AuxTypeSymValAndOff, + AuxType: AuxTypeSizeAndAlign, ArgLen: 3, FaultOnNilArg0: true, FaultOnNilArg1: true, AddrSinkArg0: true, AddrSinkArg1: true, - symEffect: SymWrite, Reg: RegInfo{ Inputs: []InputInfo{ {0, RegMask{V1: 1006632896, V2: 0}}, // X7 X8 X9 X10 X11 X12 X13 X14 X15 X16 X17 X18 X19 X20 X21 X22 X23 X24 X25 X26 X28 X29 X30 @@ -110830,6 +110826,36 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 3, Generic: true, }, + { + Name: "ConcatSaturateToInt16GroupedInt32x16", + ArgLen: 2, + Generic: true, + }, + { + Name: "ConcatSaturateToInt16GroupedInt32x8", + ArgLen: 2, + Generic: true, + }, + { + Name: "ConcatSaturateToInt16Int32x4", + ArgLen: 2, + Generic: true, + }, + { + Name: "ConcatSaturateToUint16GroupedInt32x16", + ArgLen: 2, + Generic: true, + }, + { + Name: "ConcatSaturateToUint16GroupedInt32x8", + ArgLen: 2, + Generic: true, + }, + { + Name: "ConcatSaturateToUint16Int32x4", + ArgLen: 2, + Generic: true, + }, { Name: "ConcatShiftBytesRightGroupedUint8x32", AuxType: AuxTypeUInt8, @@ -114933,21 +114959,6 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 3, Generic: true, }, - { - Name: "SaturateToInt16ConcatGroupedInt32x16", - ArgLen: 2, - Generic: true, - }, - { - Name: "SaturateToInt16ConcatGroupedInt32x8", - ArgLen: 2, - Generic: true, - }, - { - Name: "SaturateToInt16ConcatInt32x4", - ArgLen: 2, - Generic: true, - }, { Name: "SaturateToInt16Int32x16", ArgLen: 1, @@ -115038,21 +115049,6 @@ var OpcodeTable = [...]OpInfo{ ArgLen: 1, Generic: true, }, - { - Name: "SaturateToUint16ConcatGroupedInt32x16", - ArgLen: 2, - Generic: true, - }, - { - Name: "SaturateToUint16ConcatGroupedInt32x8", - ArgLen: 2, - Generic: true, - }, - { - Name: "SaturateToUint16ConcatInt32x4", - ArgLen: 2, - Generic: true, - }, { Name: "SaturateToUint16Int32x4", ArgLen: 1, diff --git a/src/cmd/compile/internal/ssa/value.go b/src/cmd/compile/internal/ssa/value.go index 05547dd5f6af10..bd5d9882999fdc 100644 --- a/src/cmd/compile/internal/ssa/value.go +++ b/src/cmd/compile/internal/ssa/value.go @@ -226,6 +226,10 @@ func (v *Value) AuxArm64ConditionalParams() Arm64ConditionalParams { return AuxIntToArm64ConditionalParams(v.AuxInt) } +func (v *Value) AuxSizeAndAlign() (int64, int64) { + return v.AuxInt, int64(v.Aux.(Int64Aux)) +} + // long form print. v# = opcode [aux] args [: reg] (names) func (v *Value) LongString() string { if v == nil { @@ -324,6 +328,8 @@ func (v *Value) AuxString() string { return fmt.Sprintf(" {%v}", v.Aux) case ssaop.AuxTypeFlagConstant: return fmt.Sprintf("[%s]", FlagConstant(v.AuxInt)) + case ssaop.AuxTypeSizeAndAlign: + return fmt.Sprintf(" [size=%d] {align=%d}", v.AuxInt, v.Aux) case ssaop.AuxTypeNone: return "" default: diff --git a/src/cmd/compile/internal/ssacompile/loopbce.go b/src/cmd/compile/internal/ssacompile/loopbce.go index 0c44e44cc2c326..24a54f0daafeb9 100644 --- a/src/cmd/compile/internal/ssacompile/loopbce.go +++ b/src/cmd/compile/internal/ssacompile/loopbce.go @@ -17,8 +17,9 @@ import ( type indVarFlags uint8 const ( - indVarMinExc indVarFlags = 1 << iota // minimum value is exclusive (default: inclusive) - indVarMaxInc // maximum value is inclusive (default: exclusive) + indVarMinExc indVarFlags = 1 << iota // minimum value is exclusive (default: inclusive) + indVarMaxInc // maximum value is inclusive (default: exclusive) + indVarDownward // downward counting loop (default: upward) ) type indVar struct { @@ -331,7 +332,7 @@ nextblock: } else { min = limit max = init - flags |= indVarMaxInc + flags |= indVarMaxInc | indVarDownward if !inclusive { flags |= indVarMinExc } @@ -353,7 +354,7 @@ nextblock: step: step, flags: flags, }) - b.Logf("found induction variable %v (inc = %v, min = %v, max = %v)\n", ind, inc, min, max) + b.Logf("found induction variable %v (inc = %v, min = %v, max = %v), downward=%t\n", ind, inc, min, max, flags&indVarDownward != 0) } } } diff --git a/src/cmd/compile/internal/ssacompile/nilcheck.go b/src/cmd/compile/internal/ssacompile/nilcheck.go index 4db3eb290831a0..c9f7b0c5ebdbc2 100644 --- a/src/cmd/compile/internal/ssacompile/nilcheck.go +++ b/src/cmd/compile/internal/ssacompile/nilcheck.go @@ -306,7 +306,7 @@ func nilcheckelim2(f *ssa.Func) { case ssaop.AuxTypeInt64: // ARM uses this auxType for duffcopy/duffzero/alignment info. // It does not affect the effective address. - case ssaop.AuxTypeNone: + case ssaop.AuxTypeNone, ssaop.AuxTypeSizeAndAlign: // offset is zero. default: v.Fatalf("can't handle aux %s (type %d) yet\n", v.AuxString(), int(ssaop.OpcodeTable[v.Op].AuxType)) diff --git a/src/cmd/compile/internal/ssacompile/prove.go b/src/cmd/compile/internal/ssacompile/prove.go index 74c1100a3e378c..a5d177ceea540a 100644 --- a/src/cmd/compile/internal/ssacompile/prove.go +++ b/src/cmd/compile/internal/ssacompile/prove.go @@ -125,6 +125,15 @@ type limitFact struct { limit ssa.Limit } +// a constDeltaAdd encodes non-over/underflowing additions like v = w + delta. +type constDeltaAdd struct { + next *constDeltaAdd + // Note: w is implicit here, determined by which additions map entry it is in (additions[w.ID]). + v *ssa.Value + delta int64 + d domain // signed or unsigned +} + // An ordering encodes facts like v < w. type ordering struct { next *ordering // linked list of all known orderings for v. @@ -157,6 +166,13 @@ type factsTable struct { orderS *ssa.Poset orderU *ssa.Poset + // additions maps a base Value ID to a linked list of known additions. + // additions[w.ID] is the list of known values v such that v = w + delta, + // where delta is a constant. + additions map[ssa.ID]*constDeltaAdd + additionsStack []ssa.ID // undo stack + additionCache *constDeltaAdd // free list + // orderings contains a list of known orderings between values. // These lists are indexed by v.ID. // We do not record transitive orderings. Only explicitly learned @@ -191,6 +207,8 @@ func newFactsTable(f *ssa.Func) *factsTable { ft := &factsTable{} ft.orderS = f.NewPoset() ft.orderU = f.NewPoset() + ft.additions = make(map[ssa.ID]*constDeltaAdd) + ft.additionsStack = make([]ssa.ID, 0, 64) ft.orderings = make(map[ssa.ID]*ordering) ft.limits = f.Cache.AllocLimitSlice(f.NumValues()) for _, b := range f.Blocks { @@ -905,6 +923,7 @@ func (ft *factsTable) checkpoint() { ft.limitStack = append(ft.limitStack, checkpointBound) ft.orderS.Checkpoint() ft.orderU.Checkpoint() + ft.additionsStack = append(ft.additionsStack, 0) ft.orderingsStack = append(ft.orderingsStack, 0) } @@ -927,6 +946,17 @@ func (ft *factsTable) restore() { } ft.orderS.Undo() ft.orderU.Undo() + for { + id := ft.additionsStack[len(ft.additionsStack)-1] + ft.additionsStack = ft.additionsStack[:len(ft.additionsStack)-1] + if id == 0 { // checkpoint marker + break + } + a := ft.additions[id] + ft.additions[id] = a.next + a.next = ft.additionCache + ft.additionCache = a + } for { id := ft.orderingsStack[len(ft.orderingsStack)-1] ft.orderingsStack = ft.orderingsStack[:len(ft.orderingsStack)-1] @@ -1207,6 +1237,7 @@ func getSliceInfo(vp *ssa.Value) (inf sliceInfo) { func prove(f *ssa.Func) { // Find induction variables. var indVars map[*ssa.Block][]indVar + var headerIndVars map[*ssa.Block][]indVar for _, v := range findIndVar(f) { ind := v.ind if len(ind.Args) != 2 { @@ -1220,8 +1251,10 @@ func prove(f *ssa.Func) { // ind or nxt is used inside the loop, add it for the facts table if indVars == nil { indVars = make(map[*ssa.Block][]indVar) + headerIndVars = make(map[*ssa.Block][]indVar) } indVars[v.entry] = append(indVars[v.entry], v) + headerIndVars[ind.Block] = append(headerIndVars[ind.Block], v) continue } else { // Since this induction variable is not used for anything but counting the iterations, @@ -1315,6 +1348,11 @@ func prove(f *ssa.Func) { addIndVarRestrictions(ft, parent, iv) } + // Entering a loop header block, add facts about the induction variables' init bounds. + for _, iv := range headerIndVars[node.block] { + addIndVarInitRestrictions(ft, parent, iv) + } + // Add results of reaching this block via a branch from // its immediate dominator (if any). if branch != unknown { @@ -1768,6 +1806,33 @@ func getBranch(sdom ssa.SparseTree, p *ssa.Block, b *ssa.Block) branch { return unknown } +// addIndVarInitRestrictions updates the factsTables ft with the init value +// learned from the induction variable indVar which drives the loop +// starting in Block b. +func addIndVarInitRestrictions(ft *factsTable, b *ssa.Block, iv indVar) { + if iv.flags&indVarDownward == 0 { + // upward counting loop, the init value is the min + d := signed + if ft.isNonNegative(iv.min) { + d |= unsigned + } + if iv.flags&indVarMinExc == 0 { + addRestrictions(b, ft, d, iv.min, iv.ind, lt|eq) + } else { + addRestrictions(b, ft, d, iv.min, iv.ind, lt) + } + } else { + // downward counting loop, the init value is the max. + // We must only use signed domain because iv.ind can become + // negative on the exit iteration, violating unsigned iv.ind <= iv.max. + if iv.flags&indVarMaxInc == 0 { + addRestrictions(b, ft, signed, iv.ind, iv.max, lt) + } else { + addRestrictions(b, ft, signed, iv.ind, iv.max, lt|eq) + } + } +} + // addIndVarRestrictions updates the factsTables ft with the facts // learned from the induction variable indVar which drives the loop // starting in Block b. @@ -1980,6 +2045,7 @@ func (ft *factsTable) addValueFact(b *ssa.Block, v *ssa.Value) { } ft.update(b, v, v.Args[0], signed, r) } + ft.compareConstDelta(b, v) case ssaop.OpSub64, ssaop.OpSub32, ssaop.OpSub16, ssaop.OpSub8: x := ft.limits[v.Args[0].ID] y := ft.limits[v.Args[1].ID] @@ -1990,6 +2056,7 @@ func (ft *factsTable) addValueFact(b *ssa.Block, v *ssa.Value) { } ft.update(b, v, v.Args[0], unsigned, r) } + ft.compareConstDelta(b, v) // FIXME: we could also do signed facts but the overflow checks are much trickier and I don't need it yet. case ssaop.OpAnd64, ssaop.OpAnd32, ssaop.OpAnd16, ssaop.OpAnd8: ft.update(b, v, v.Args[0], unsigned, lt|eq) @@ -2641,6 +2708,72 @@ func isConstDelta(v *ssa.Value) (w *ssa.Value, delta int64) { return nil, 0 } +// recordAddition records a relationship v = w + delta for domain d, delta can be negative. +// This addition must be non-over/underflowing. +func (ft *factsTable) recordAddition(w *ssa.Value, v *ssa.Value, delta int64, d domain) { + a := ft.additionCache + if a == nil { + a = &constDeltaAdd{} + } else { + ft.additionCache = a.next + } + a.v = v + a.delta = delta + a.d = d + a.next = ft.additions[w.ID] + ft.additions[w.ID] = a + ft.additionsStack = append(ft.additionsStack, w.ID) +} + +// compareConstDelta tries to add a relation between v1 = w + delta1 and v2 = w + delta2. +// This function also records this addition. +func (ft *factsTable) compareConstDelta(b *ssa.Block, v1 *ssa.Value) { + w, delta1 := isConstDelta(v1) + if w == nil { + return + } + domains := make([]domain, 0) + // Check for over/underflows. + // unsigned domain + lim := ft.limits[w.ID] + if (delta1 > 0 && !unsignedAddOverflows(lim.Umax, uint64(delta1), w.Type)) || + (delta1 < 0 && !unsignedSubUnderflows(lim.Umin, uint64(-delta1))) { + domains = append(domains, unsigned) + } + // signed domain + if !signedAddOverflowsOrUnderflows(lim.Max, delta1, w.Type) && + !signedAddOverflowsOrUnderflows(lim.Min, delta1, w.Type) { + domains = append(domains, signed) + } + for _, d := range domains { + var bestLowerBound, bestUpperBound *ssa.Value + maxDelta := int64(math.MinInt64) + minDelta := int64(math.MaxInt64) + + for a := ft.additions[w.ID]; a != nil; a = a.next { + if a.d != d { + continue + } + delta2 := a.delta + // pick the tightest delta2 to avoid quadratic comparisons. + if delta2 < delta1 && delta2 > maxDelta { + bestLowerBound = a.v + maxDelta = delta2 + } else if delta2 > delta1 && delta2 < minDelta { + bestUpperBound = a.v + minDelta = delta2 + } + } + if bestLowerBound != nil { + ft.update(b, v1, bestLowerBound, d, gt) + } + if bestUpperBound != nil { + ft.update(b, bestUpperBound, v1, d, gt) + } + ft.recordAddition(w, v1, delta1, d) + } +} + // isCleanExt reports whether v is the result of a value-preserving // sign or zero extension. func isCleanExt(v *ssa.Value) bool { diff --git a/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go b/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go index 545c5a3ef5ce25..a79d0ce1aaaac1 100644 --- a/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go +++ b/src/cmd/compile/internal/ssagen/simdAMD64intrinsics.go @@ -237,6 +237,12 @@ func simdAMD64Intrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFamil addF(simdPackage, "Uint16x16.ConcatPermute128Scalars", opLen2Imm8_II(ssaop.OpConcatPermute128ScalarsUint16x16, types.TypeVec256, 0), sys.AMD64) addF(simdPackage, "Uint32x8.ConcatPermute128Scalars", opLen2Imm8_II(ssaop.OpConcatPermute128ScalarsUint32x8, types.TypeVec256, 0), sys.AMD64) addF(simdPackage, "Uint64x4.ConcatPermute128Scalars", opLen2Imm8_II(ssaop.OpConcatPermute128ScalarsUint64x4, types.TypeVec256, 0), sys.AMD64) + addF(simdPackage, "Int32x4.ConcatSaturateToInt16", opLen2(ssaop.OpConcatSaturateToInt16Int32x4, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Int32x8.ConcatSaturateToInt16Grouped", opLen2(ssaop.OpConcatSaturateToInt16GroupedInt32x8, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Int32x16.ConcatSaturateToInt16Grouped", opLen2(ssaop.OpConcatSaturateToInt16GroupedInt32x16, types.TypeVec512), sys.AMD64) + addF(simdPackage, "Int32x4.ConcatSaturateToUint16", opLen2(ssaop.OpConcatSaturateToUint16Int32x4, types.TypeVec128), sys.AMD64) + addF(simdPackage, "Int32x8.ConcatSaturateToUint16Grouped", opLen2(ssaop.OpConcatSaturateToUint16GroupedInt32x8, types.TypeVec256), sys.AMD64) + addF(simdPackage, "Int32x16.ConcatSaturateToUint16Grouped", opLen2(ssaop.OpConcatSaturateToUint16GroupedInt32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint8x16.ConcatShiftBytesRight", opLen2Imm8_2I(ssaop.OpConcatShiftBytesRightUint8x16, types.TypeVec128, 0), sys.AMD64) addF(simdPackage, "Uint8x32.ConcatShiftBytesRightGrouped", opLen2Imm8_2I(ssaop.OpConcatShiftBytesRightGroupedUint8x32, types.TypeVec256, 0), sys.AMD64) addF(simdPackage, "Uint8x64.ConcatShiftBytesRightGrouped", opLen2Imm8_2I(ssaop.OpConcatShiftBytesRightGroupedUint8x64, types.TypeVec512, 0), sys.AMD64) @@ -894,9 +900,6 @@ func simdAMD64Intrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFamil addF(simdPackage, "Int64x2.SaturateToInt16", opLen1(ssaop.OpSaturateToInt16Int64x2, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x4.SaturateToInt16", opLen1(ssaop.OpSaturateToInt16Int64x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x8.SaturateToInt16", opLen1(ssaop.OpSaturateToInt16Int64x8, types.TypeVec128), sys.AMD64) - addF(simdPackage, "Int32x4.SaturateToInt16Concat", opLen2(ssaop.OpSaturateToInt16ConcatInt32x4, types.TypeVec128), sys.AMD64) - addF(simdPackage, "Int32x8.SaturateToInt16ConcatGrouped", opLen2(ssaop.OpSaturateToInt16ConcatGroupedInt32x8, types.TypeVec256), sys.AMD64) - addF(simdPackage, "Int32x16.SaturateToInt16ConcatGrouped", opLen2(ssaop.OpSaturateToInt16ConcatGroupedInt32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Int64x2.SaturateToInt32", opLen1(ssaop.OpSaturateToInt32Int64x2, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x4.SaturateToInt32", opLen1(ssaop.OpSaturateToInt32Int64x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Int64x8.SaturateToInt32", opLen1(ssaop.OpSaturateToInt32Int64x8, types.TypeVec256), sys.AMD64) @@ -915,9 +918,6 @@ func simdAMD64Intrinsics(addF func(pkg, fn string, b intrinsicBuilder, archFamil addF(simdPackage, "Uint64x2.SaturateToUint16", opLen1(ssaop.OpSaturateToUint16Uint64x2, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x4.SaturateToUint16", opLen1(ssaop.OpSaturateToUint16Uint64x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x8.SaturateToUint16", opLen1(ssaop.OpSaturateToUint16Uint64x8, types.TypeVec128), sys.AMD64) - addF(simdPackage, "Int32x4.SaturateToUint16Concat", opLen2(ssaop.OpSaturateToUint16ConcatInt32x4, types.TypeVec128), sys.AMD64) - addF(simdPackage, "Int32x8.SaturateToUint16ConcatGrouped", opLen2(ssaop.OpSaturateToUint16ConcatGroupedInt32x8, types.TypeVec256), sys.AMD64) - addF(simdPackage, "Int32x16.SaturateToUint16ConcatGrouped", opLen2(ssaop.OpSaturateToUint16ConcatGroupedInt32x16, types.TypeVec512), sys.AMD64) addF(simdPackage, "Uint64x2.SaturateToUint32", opLen1(ssaop.OpSaturateToUint32Uint64x2, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x4.SaturateToUint32", opLen1(ssaop.OpSaturateToUint32Uint64x4, types.TypeVec128), sys.AMD64) addF(simdPackage, "Uint64x8.SaturateToUint32", opLen1(ssaop.OpSaturateToUint32Uint64x8, types.TypeVec256), sys.AMD64) diff --git a/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go b/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go index 14254c4423a688..5bc4a1bfdbac82 100644 --- a/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go +++ b/src/cmd/compile/internal/ssarewrite/rewriteamd64/rewriteAMD64.go @@ -3817,6 +3817,24 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpConcatPermuteUint8x64: v.Op = ssaop.OpAMD64VPERMI2B512 return true + case ssaop.OpConcatSaturateToInt16GroupedInt32x16: + v.Op = ssaop.OpAMD64VPACKSSDW512 + return true + case ssaop.OpConcatSaturateToInt16GroupedInt32x8: + v.Op = ssaop.OpAMD64VPACKSSDW256 + return true + case ssaop.OpConcatSaturateToInt16Int32x4: + v.Op = ssaop.OpAMD64VPACKSSDW128 + return true + case ssaop.OpConcatSaturateToUint16GroupedInt32x16: + v.Op = ssaop.OpAMD64VPACKUSDW512 + return true + case ssaop.OpConcatSaturateToUint16GroupedInt32x8: + v.Op = ssaop.OpAMD64VPACKUSDW256 + return true + case ssaop.OpConcatSaturateToUint16Int32x4: + v.Op = ssaop.OpAMD64VPACKUSDW128 + return true case ssaop.OpConcatShiftBytesRightGroupedUint8x32: v.Op = ssaop.OpAMD64VPALIGNR256 return true @@ -6070,15 +6088,6 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpSHA256TwoRoundsUint32x4: v.Op = ssaop.OpAMD64SHA256RNDS2128 return true - case ssaop.OpSaturateToInt16ConcatGroupedInt32x16: - v.Op = ssaop.OpAMD64VPACKSSDW512 - return true - case ssaop.OpSaturateToInt16ConcatGroupedInt32x8: - v.Op = ssaop.OpAMD64VPACKSSDW256 - return true - case ssaop.OpSaturateToInt16ConcatInt32x4: - v.Op = ssaop.OpAMD64VPACKSSDW128 - return true case ssaop.OpSaturateToInt16Int32x16: v.Op = ssaop.OpAMD64VPMOVSDW256 return true @@ -6133,15 +6142,6 @@ func RewriteValue(v *ssa.Value) bool { case ssaop.OpSaturateToInt8Int64x8: v.Op = ssaop.OpAMD64VPMOVSQB128_512 return true - case ssaop.OpSaturateToUint16ConcatGroupedInt32x16: - v.Op = ssaop.OpAMD64VPACKUSDW512 - return true - case ssaop.OpSaturateToUint16ConcatGroupedInt32x8: - v.Op = ssaop.OpAMD64VPACKUSDW256 - return true - case ssaop.OpSaturateToUint16ConcatInt32x4: - v.Op = ssaop.OpAMD64VPACKUSDW128 - return true case ssaop.OpSaturateToUint16Uint32x16: v.Op = ssaop.OpAMD64VPMOVUSDW256 return true @@ -50809,6 +50809,32 @@ func rewriteValue_OpAMD64VMOVDQU32Masked128(v *ssa.Value) bool { v.AddArg4(x, y, z, mask) return true } + // match: (VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) + // result: (VPACKSSDWMasked128 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKSSDW128 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKSSDWMasked128) + v.AddArg3(x, y, mask) + return true + } + // match: (VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) + // result: (VPACKUSDWMasked128 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKUSDW128 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKUSDWMasked128) + v.AddArg3(x, y, mask) + return true + } // match: (VMOVDQU32Masked128 (VCVTDQ2PS128 x) mask) // result: (VCVTDQ2PSMasked128 x mask) for { @@ -51102,19 +51128,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked128(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked128 (VPACKSSDW128 x y) mask) - // result: (VPACKSSDWMasked128 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKSSDW128 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKSSDWMasked128) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked128 (VPMOVSDW128_128 x) mask) // result: (VPMOVSDWMasked128_128 x mask) for { @@ -51139,19 +51152,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked128(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked128 (VPACKUSDW128 x y) mask) - // result: (VPACKUSDWMasked128 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKUSDW128 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKUSDWMasked128) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked128 (VPMOVUSDW128_128 x) mask) // result: (VPMOVUSDWMasked128_128 x mask) for { @@ -51530,6 +51530,32 @@ func rewriteValue_OpAMD64VMOVDQU32Masked256(v *ssa.Value) bool { v.AddArg4(x, y, z, mask) return true } + // match: (VMOVDQU32Masked256 (VPACKSSDW256 x y) mask) + // result: (VPACKSSDWMasked256 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKSSDW256 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKSSDWMasked256) + v.AddArg3(x, y, mask) + return true + } + // match: (VMOVDQU32Masked256 (VPACKUSDW256 x y) mask) + // result: (VPACKUSDWMasked256 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKUSDW256 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKUSDWMasked256) + v.AddArg3(x, y, mask) + return true + } // match: (VMOVDQU32Masked256 (VCVTDQ2PS256 x) mask) // result: (VCVTDQ2PSMasked256 x mask) for { @@ -51909,19 +51935,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked256(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked256 (VPACKSSDW256 x y) mask) - // result: (VPACKSSDWMasked256 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKSSDW256 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKSSDWMasked256) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked256 (VPMOVSDW128_256 x) mask) // result: (VPMOVSDWMasked128_256 x mask) for { @@ -51958,19 +51971,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked256(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked256 (VPACKUSDW256 x y) mask) - // result: (VPACKUSDWMasked256 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKUSDW256 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKUSDWMasked256) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked256 (VPMOVUSDW128_256 x) mask) // result: (VPMOVUSDWMasked128_256 x mask) for { @@ -52399,6 +52399,32 @@ func rewriteValue_OpAMD64VMOVDQU32Masked512(v *ssa.Value) bool { v.AddArg4(x, y, z, mask) return true } + // match: (VMOVDQU32Masked512 (VPACKSSDW512 x y) mask) + // result: (VPACKSSDWMasked512 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKSSDW512 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKSSDWMasked512) + v.AddArg3(x, y, mask) + return true + } + // match: (VMOVDQU32Masked512 (VPACKUSDW512 x y) mask) + // result: (VPACKUSDWMasked512 x y mask) + for { + if v_0.Op != ssaop.OpAMD64VPACKUSDW512 { + break + } + y := v_0.Args[1] + x := v_0.Args[0] + mask := v_1 + v.Reset(ssaop.OpAMD64VPACKUSDWMasked512) + v.AddArg3(x, y, mask) + return true + } // match: (VMOVDQU32Masked512 (VCVTDQ2PS512 x) mask) // result: (VCVTDQ2PSMasked512 x mask) for { @@ -52815,19 +52841,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked512(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked512 (VPACKSSDW512 x y) mask) - // result: (VPACKSSDWMasked512 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKSSDW512 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKSSDWMasked512) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked512 (VPMOVUSDB128_512 x) mask) // result: (VPMOVUSDBMasked128_512 x mask) for { @@ -52840,19 +52853,6 @@ func rewriteValue_OpAMD64VMOVDQU32Masked512(v *ssa.Value) bool { v.AddArg2(x, mask) return true } - // match: (VMOVDQU32Masked512 (VPACKUSDW512 x y) mask) - // result: (VPACKUSDWMasked512 x y mask) - for { - if v_0.Op != ssaop.OpAMD64VPACKUSDW512 { - break - } - y := v_0.Args[1] - x := v_0.Args[0] - mask := v_1 - v.Reset(ssaop.OpAMD64VPACKUSDWMasked512) - v.AddArg3(x, y, mask) - return true - } // match: (VMOVDQU32Masked512 (VSCALEFPS512 x y) mask) // result: (VSCALEFPSMasked512 x y mask) for { diff --git a/src/cmd/compile/internal/ssarewrite/rewriteriscv64/rewriteRISCV64.go b/src/cmd/compile/internal/ssarewrite/rewriteriscv64/rewriteRISCV64.go index 5353d78bc01702..240e3c36af9083 100644 --- a/src/cmd/compile/internal/ssarewrite/rewriteriscv64/rewriteRISCV64.go +++ b/src/cmd/compile/internal/ssarewrite/rewriteriscv64/rewriteRISCV64.go @@ -3121,7 +3121,7 @@ func rewriteValue_OpMove(v *ssa.Value) bool { } // match: (Move [s] {t} dst src mem) // cond: s > 0 && s <= 3*8*ssa.MoveSize(t.Alignment(), config) && ssa.LogLargeCopyValue(v, s) - // result: (LoweredMove [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] dst src mem) + // result: (LoweredMove [s] {t.Alignment()} dst src mem) for { s := ssa.AuxIntToInt64(v.AuxInt) t := ssa.AuxToType(v.Aux) @@ -3132,13 +3132,14 @@ func rewriteValue_OpMove(v *ssa.Value) bool { break } v.Reset(ssaop.OpRISCV64LoweredMove) - v.AuxInt = ssa.ValAndOffToAuxInt(ssa.MakeValAndOff(int32(s), int32(t.Alignment()))) + v.AuxInt = ssa.Int64ToAuxInt(s) + v.Aux = ssa.Int64ToAux(t.Alignment()) v.AddArg3(dst, src, mem) return true } // match: (Move [s] {t} dst src mem) // cond: s > 3*8*ssa.MoveSize(t.Alignment(), config) && ssa.LogLargeCopyValue(v, s) - // result: (LoweredMoveLoop [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] dst src mem) + // result: (LoweredMoveLoop [s] {t.Alignment()} dst src mem) for { s := ssa.AuxIntToInt64(v.AuxInt) t := ssa.AuxToType(v.Aux) @@ -3149,7 +3150,8 @@ func rewriteValue_OpMove(v *ssa.Value) bool { break } v.Reset(ssaop.OpRISCV64LoweredMoveLoop) - v.AuxInt = ssa.ValAndOffToAuxInt(ssa.MakeValAndOff(int32(s), int32(t.Alignment()))) + v.AuxInt = ssa.Int64ToAuxInt(s) + v.Aux = ssa.Int64ToAux(t.Alignment()) v.AddArg3(dst, src, mem) return true } @@ -11088,7 +11090,7 @@ func rewriteValue_OpZero(v *ssa.Value) bool { } // match: (Zero [s] {t} ptr mem) // cond: s <= 24*ssa.MoveSize(t.Alignment(), config) - // result: (LoweredZero [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] ptr mem) + // result: (LoweredZero [s] {t.Alignment()} ptr mem) for { s := ssa.AuxIntToInt64(v.AuxInt) t := ssa.AuxToType(v.Aux) @@ -11098,13 +11100,14 @@ func rewriteValue_OpZero(v *ssa.Value) bool { break } v.Reset(ssaop.OpRISCV64LoweredZero) - v.AuxInt = ssa.ValAndOffToAuxInt(ssa.MakeValAndOff(int32(s), int32(t.Alignment()))) + v.AuxInt = ssa.Int64ToAuxInt(s) + v.Aux = ssa.Int64ToAux(t.Alignment()) v.AddArg2(ptr, mem) return true } // match: (Zero [s] {t} ptr mem) // cond: s > 24*ssa.MoveSize(t.Alignment(), config) - // result: (LoweredZeroLoop [ssa.MakeValAndOff(int32(s),int32(t.Alignment()))] ptr mem) + // result: (LoweredZeroLoop [s] {t.Alignment()} ptr mem) for { s := ssa.AuxIntToInt64(v.AuxInt) t := ssa.AuxToType(v.Aux) @@ -11114,7 +11117,8 @@ func rewriteValue_OpZero(v *ssa.Value) bool { break } v.Reset(ssaop.OpRISCV64LoweredZeroLoop) - v.AuxInt = ssa.ValAndOffToAuxInt(ssa.MakeValAndOff(int32(s), int32(t.Alignment()))) + v.AuxInt = ssa.Int64ToAuxInt(s) + v.Aux = ssa.Int64ToAux(t.Alignment()) v.AddArg2(ptr, mem) return true } diff --git a/src/encoding/json/v2_decode_test.go b/src/encoding/json/v2_decode_test.go index db83f06a6080dd..d365e6a597080a 100644 --- a/src/encoding/json/v2_decode_test.go +++ b/src/encoding/json/v2_decode_test.go @@ -2073,7 +2073,7 @@ func TestNullString(t *testing.T) { case s.B != 1: t.Fatalf("Unmarshal: s.B = %d, want 1", s.B) case s.C != nil: - t.Fatalf("Unmarshal: s.C = %d, want non-nil", s.C) + t.Fatalf("Unmarshal: s.C = new(%d), want nil", *s.C) } } diff --git a/src/simd/archsimd/_gen/simdgen/ops/Converts/categories.yaml b/src/simd/archsimd/_gen/simdgen/ops/Converts/categories.yaml index c72815f756b484..0ee08a85c066aa 100644 --- a/src/simd/archsimd/_gen/simdgen/ops/Converts/categories.yaml +++ b/src/simd/archsimd/_gen/simdgen/ops/Converts/categories.yaml @@ -69,7 +69,7 @@ regexpTag: "convert" documentation: !string |- // NAME truncates element values to int16. -- go: "SaturateToInt16(Concat(Grouped)?)?" +- go: "(Concat)?SaturateToInt16(Grouped)?" commutative: false regexpTag: "convert" documentation: !string |- @@ -114,7 +114,7 @@ regexpTag: "convert" documentation: !string |- // NAME truncates element values to uint16. -- go: "SaturateToUint16(Concat(Grouped)?)?" +- go: "(Concat)?SaturateToUint16(Grouped)?" commutative: false regexpTag: "convert" documentation: !string |- diff --git a/src/simd/archsimd/_gen/simdgen/ops/Converts/go_amd64.yaml b/src/simd/archsimd/_gen/simdgen/ops/Converts/go_amd64.yaml index e6869b6cf6d511..86de5cdba450d9 100644 --- a/src/simd/archsimd/_gen/simdgen/ops/Converts/go_amd64.yaml +++ b/src/simd/archsimd/_gen/simdgen/ops/Converts/go_amd64.yaml @@ -441,7 +441,7 @@ out: - base: uint # Truncating saturated packed -- go: SaturateToInt16Concat +- go: ConcatSaturateToInt16 regexpTag: "convert" asm: "VPACKSSDW" addDoc: &satDocConcat @@ -454,7 +454,7 @@ out: - base: int bits: 128 -- go: SaturateToInt16ConcatGrouped +- go: ConcatSaturateToInt16Grouped regexpTag: "convert" asm: "VPACKSSDW" addDoc: &satDocConcatGrouped @@ -468,7 +468,7 @@ out: - base: int bits: 256|512 -- go: SaturateToUint16Concat +- go: ConcatSaturateToUint16 regexpTag: "convert" asm: "VPACKUSDW" addDoc: *satDocConcat @@ -478,7 +478,7 @@ out: - base: uint bits: 128 -- go: SaturateToUint16ConcatGrouped +- go: ConcatSaturateToUint16Grouped regexpTag: "convert" asm: "VPACKUSDW" addDoc: *satDocConcatGrouped diff --git a/src/simd/archsimd/internal/simd_test/simd_amd64_test.go b/src/simd/archsimd/internal/simd_test/simd_amd64_test.go index 5ec9dbe49302e8..73656d4ccfdca4 100644 --- a/src/simd/archsimd/internal/simd_test/simd_amd64_test.go +++ b/src/simd/archsimd/internal/simd_test/simd_amd64_test.go @@ -1295,56 +1295,56 @@ func convConcatGroupedSlice[T, U number](a, b []T, conv func(T) U) []U { } func TestSaturateConcat(t *testing.T) { - // Int32x4.SaturateToInt16Concat + // Int32x4.ConcatSaturateToInt16 forSlicePair(t, int32s, 4, func(x, y []int32) bool { a, b := archsimd.LoadInt32x4(x), archsimd.LoadInt32x4(y) var out [8]int16 - a.SaturateToInt16Concat(b).StoreArray(&out) + a.ConcatSaturateToInt16(b).StoreArray(&out) want := convConcatSlice(x, y, satToInt16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) - // Int32x4.SaturateToUint16Concat + // Int32x4.ConcatSaturateToUint16 forSlicePair(t, int32s, 4, func(x, y []int32) bool { a, b := archsimd.LoadInt32x4(x), archsimd.LoadInt32x4(y) var out [8]uint16 - a.SaturateToUint16Concat(b).StoreArray(&out) + a.ConcatSaturateToUint16(b).StoreArray(&out) want := convConcatSlice(x, y, satToUint16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) if archsimd.X86.AVX2() { - // Int32x8.SaturateToInt16ConcatGrouped + // Int32x8.ConcatSaturateToInt16Grouped forSlicePair(t, int32s, 8, func(x, y []int32) bool { a, b := archsimd.LoadInt32x8(x), archsimd.LoadInt32x8(y) var out [16]int16 - a.SaturateToInt16ConcatGrouped(b).StoreArray(&out) + a.ConcatSaturateToInt16Grouped(b).StoreArray(&out) want := convConcatGroupedSlice(x, y, satToInt16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) - // Int32x8.SaturateToUint16ConcatGrouped + // Int32x8.ConcatSaturateToUint16Grouped forSlicePair(t, int32s, 8, func(x, y []int32) bool { a, b := archsimd.LoadInt32x8(x), archsimd.LoadInt32x8(y) var out [16]uint16 - a.SaturateToUint16ConcatGrouped(b).StoreArray(&out) + a.ConcatSaturateToUint16Grouped(b).StoreArray(&out) want := convConcatGroupedSlice(x, y, satToUint16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) } if archsimd.X86.AVX512() { - // Int32x16.SaturateToInt16ConcatGrouped + // Int32x16.ConcatSaturateToInt16Grouped forSlicePair(t, int32s, 16, func(x, y []int32) bool { a, b := archsimd.LoadInt32x16(x), archsimd.LoadInt32x16(y) var out [32]int16 - a.SaturateToInt16ConcatGrouped(b).StoreArray(&out) + a.ConcatSaturateToInt16Grouped(b).StoreArray(&out) want := convConcatGroupedSlice(x, y, satToInt16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) - // Int32x16.SaturateToUint16ConcatGrouped + // Int32x16.ConcatSaturateToUint16Grouped forSlicePair(t, int32s, 16, func(x, y []int32) bool { a, b := archsimd.LoadInt32x16(x), archsimd.LoadInt32x16(y) var out [32]uint16 - a.SaturateToUint16ConcatGrouped(b).StoreArray(&out) + a.ConcatSaturateToUint16Grouped(b).StoreArray(&out) want := convConcatGroupedSlice(x, y, satToUint16) return checkSlicesLogInput(t, out[:], want, 0, func() { t.Logf("x=%v, y=%v", x, y) }) }) diff --git a/src/simd/archsimd/internal/simd_test/simd_test.go b/src/simd/archsimd/internal/simd_test/simd_test.go index 47cf989996346a..908fbb4e6e02a9 100644 --- a/src/simd/archsimd/internal/simd_test/simd_test.go +++ b/src/simd/archsimd/internal/simd_test/simd_test.go @@ -8,6 +8,7 @@ package simd_test import ( "reflect" + "simd" "simd/archsimd" "testing" ) @@ -377,3 +378,9 @@ func TestSlicesInt8SetElem17const(t *testing.T) { e := v.SetElem(17, 18).GetElem(2) t.Errorf("Should have panicked, e=%v", e) } + +func TestIssue81264(t *testing.T) { + var _ simd.Float32s + for range 1 { + } +} diff --git a/src/simd/archsimd/ops_amd64.go b/src/simd/archsimd/ops_amd64.go index f93313dfdc0bf9..a93dc4df5132d3 100644 --- a/src/simd/archsimd/ops_amd64.go +++ b/src/simd/archsimd/ops_amd64.go @@ -1565,6 +1565,60 @@ func (x Uint32x8) ConcatPermute128Scalars(lo, hi uint8, y Uint32x8) Uint32x8 // Asm: VPERM2I128, CPU Feature: AVX2 func (x Uint64x4) ConcatPermute128Scalars(lo, hi uint8, y Uint64x4) Uint64x4 +/* ConcatSaturateToInt16 */ + +// ConcatSaturateToInt16 converts element values to int16 with signed saturation. +// The converted elements from x will be packed to the lower part of the result vector, +// the converted elements from y will be packed to the upper part of the result vector. +// +// Asm: VPACKSSDW, CPU Feature: AVX +func (x Int32x4) ConcatSaturateToInt16(y Int32x4) Int16x8 + +/* ConcatSaturateToInt16Grouped */ + +// ConcatSaturateToInt16Grouped converts element values to int16 with signed saturation. +// With each 128-bit as a group: +// The converted elements from x will be packed to the lower part of the group in the result vector, +// the converted elements from y will be packed to the upper part of the group in the result vector. +// +// Asm: VPACKSSDW, CPU Feature: AVX2 +func (x Int32x8) ConcatSaturateToInt16Grouped(y Int32x8) Int16x16 + +// ConcatSaturateToInt16Grouped converts element values to int16 with signed saturation. +// With each 128-bit as a group: +// The converted elements from x will be packed to the lower part of the group in the result vector, +// the converted elements from y will be packed to the upper part of the group in the result vector. +// +// Asm: VPACKSSDW, CPU Feature: AVX512 +func (x Int32x16) ConcatSaturateToInt16Grouped(y Int32x16) Int16x32 + +/* ConcatSaturateToUint16 */ + +// ConcatSaturateToUint16 converts element values to uint16 with unsigned saturation. +// The converted elements from x will be packed to the lower part of the result vector, +// the converted elements from y will be packed to the upper part of the result vector. +// +// Asm: VPACKUSDW, CPU Feature: AVX +func (x Int32x4) ConcatSaturateToUint16(y Int32x4) Uint16x8 + +/* ConcatSaturateToUint16Grouped */ + +// ConcatSaturateToUint16Grouped converts element values to uint16 with unsigned saturation. +// With each 128-bit as a group: +// The converted elements from x will be packed to the lower part of the group in the result vector, +// the converted elements from y will be packed to the upper part of the group in the result vector. +// +// Asm: VPACKUSDW, CPU Feature: AVX2 +func (x Int32x8) ConcatSaturateToUint16Grouped(y Int32x8) Uint16x16 + +// ConcatSaturateToUint16Grouped converts element values to uint16 with unsigned saturation. +// With each 128-bit as a group: +// The converted elements from x will be packed to the lower part of the group in the result vector, +// the converted elements from y will be packed to the upper part of the group in the result vector. +// +// Asm: VPACKUSDW, CPU Feature: AVX512 +func (x Int32x16) ConcatSaturateToUint16Grouped(y Int32x16) Uint16x32 + /* ConcatShiftBytesRight */ // ConcatShiftBytesRight concatenates x and y and shifts it right by shift bytes. @@ -5369,33 +5423,6 @@ func (x Int64x4) SaturateToInt16() Int16x8 // Asm: VPMOVSQW, CPU Feature: AVX512 func (x Int64x8) SaturateToInt16() Int16x8 -/* SaturateToInt16Concat */ - -// SaturateToInt16Concat converts element values to int16 with signed saturation. -// The converted elements from x will be packed to the lower part of the result vector, -// the converted elements from y will be packed to the upper part of the result vector. -// -// Asm: VPACKSSDW, CPU Feature: AVX -func (x Int32x4) SaturateToInt16Concat(y Int32x4) Int16x8 - -/* SaturateToInt16ConcatGrouped */ - -// SaturateToInt16ConcatGrouped converts element values to int16 with signed saturation. -// With each 128-bit as a group: -// The converted elements from x will be packed to the lower part of the group in the result vector, -// the converted elements from y will be packed to the upper part of the group in the result vector. -// -// Asm: VPACKSSDW, CPU Feature: AVX2 -func (x Int32x8) SaturateToInt16ConcatGrouped(y Int32x8) Int16x16 - -// SaturateToInt16ConcatGrouped converts element values to int16 with signed saturation. -// With each 128-bit as a group: -// The converted elements from x will be packed to the lower part of the group in the result vector, -// the converted elements from y will be packed to the upper part of the group in the result vector. -// -// Asm: VPACKSSDW, CPU Feature: AVX512 -func (x Int32x16) SaturateToInt16ConcatGrouped(y Int32x16) Int16x32 - /* SaturateToInt32 */ // SaturateToInt32 converts element values to int32 with signed saturation. @@ -5502,33 +5529,6 @@ func (x Uint64x4) SaturateToUint16() Uint16x8 // Asm: VPMOVUSQW, CPU Feature: AVX512 func (x Uint64x8) SaturateToUint16() Uint16x8 -/* SaturateToUint16Concat */ - -// SaturateToUint16Concat converts element values to uint16 with unsigned saturation. -// The converted elements from x will be packed to the lower part of the result vector, -// the converted elements from y will be packed to the upper part of the result vector. -// -// Asm: VPACKUSDW, CPU Feature: AVX -func (x Int32x4) SaturateToUint16Concat(y Int32x4) Uint16x8 - -/* SaturateToUint16ConcatGrouped */ - -// SaturateToUint16ConcatGrouped converts element values to uint16 with unsigned saturation. -// With each 128-bit as a group: -// The converted elements from x will be packed to the lower part of the group in the result vector, -// the converted elements from y will be packed to the upper part of the group in the result vector. -// -// Asm: VPACKUSDW, CPU Feature: AVX2 -func (x Int32x8) SaturateToUint16ConcatGrouped(y Int32x8) Uint16x16 - -// SaturateToUint16ConcatGrouped converts element values to uint16 with unsigned saturation. -// With each 128-bit as a group: -// The converted elements from x will be packed to the lower part of the group in the result vector, -// the converted elements from y will be packed to the upper part of the group in the result vector. -// -// Asm: VPACKUSDW, CPU Feature: AVX512 -func (x Int32x16) SaturateToUint16ConcatGrouped(y Int32x16) Uint16x32 - /* SaturateToUint32 */ // SaturateToUint32 converts element values to uint32 with unsigned saturation. diff --git a/test/codegen/comparisons.go b/test/codegen/comparisons.go index b5cb27c31ffb32..d81498305b4ef4 100644 --- a/test/codegen/comparisons.go +++ b/test/codegen/comparisons.go @@ -307,58 +307,58 @@ func CmpLogicalToZero(a, b, c uint32, d, e, f, g uint64) uint64 { // var + const // 'x-const' might be canonicalized to 'x+(-const)', so we check both // CMN and CMP for subtraction expressions to make the pattern robust. -func CmpToZero_ex1(a int64, e int32) int { +func CmpToZero_ex1(a [6]int64, e [4]int32) int { // arm64:`CMN` -`ADD` `(BMI|BPL)` - if a+3 < 0 { + if a[0]+3 < 0 { return 1 } // arm64:`CMN` -`ADD` `BEQ` `(BMI|BPL)` - if a+5 <= 0 { + if a[1]+5 <= 0 { return 1 } // arm64:`CMN` -`ADD` `(BMI|BPL)` - if a+13 >= 0 { + if a[2]+13 >= 0 { return 2 } // arm64:`CMP|CMN` -`(ADD|SUB)` `(BMI|BPL)` - if a-7 < 0 { + if a[3]-7 < 0 { return 3 } // arm64:`SUB` `TBZ` - if a-11 >= 0 { + if a[4]-11 >= 0 { return 4 } // arm64:`SUB` `CMP` `BGT` - if a-19 > 0 { + if a[5]-19 > 0 { return 4 } // arm64:`CMNW` -`ADDW` `(BMI|BPL)` // arm:`CMN` -`ADD` `(BMI|BPL)` - if e+3 < 0 { + if e[0]+3 < 0 { return 5 } // arm64:`CMNW` -`ADDW` `(BMI|BPL)` // arm:`CMN` -`ADD` `(BMI|BPL)` - if e+13 >= 0 { + if e[1]+13 >= 0 { return 6 } // arm64:`CMPW|CMNW` `(BMI|BPL)` // arm:`CMP|CMN` -`(ADD|SUB)` `(BMI|BPL)` - if e-7 < 0 { + if e[2]-7 < 0 { return 7 } // arm64:`SUB` `TBNZ` // arm:`SUB` -`(BMI|BPL)` - if e-11 >= 0 { + if e[3]-11 >= 0 { return 8 } diff --git a/test/fixedbugs/issue81240.go b/test/fixedbugs/issue81240.go new file mode 100644 index 00000000000000..a98e89048d258b --- /dev/null +++ b/test/fixedbugs/issue81240.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +const W = 32 << (^uintptr(0) >> 63) // 32 or 64 + +type T struct { + a [1<<(W-30) - 1]byte +} + +func f(x, y *T) { + *x = *y +} diff --git a/test/fixedbugs/issue81242.go b/test/fixedbugs/issue81242.go new file mode 100644 index 00000000000000..c9cacce29bccfa --- /dev/null +++ b/test/fixedbugs/issue81242.go @@ -0,0 +1,17 @@ +// compile + +// Copyright 2026 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package p + +const W = 32 << (^uintptr(0) >> 63) // 32 or 64 + +type T struct { + a [1<<(W-30) - 1]byte +} + +func f(t *T) { + *t = T{} +} diff --git a/test/prove.go b/test/prove.go index 07747e1a6010b2..a384c438fa094b 100644 --- a/test/prove.go +++ b/test/prove.go @@ -796,7 +796,7 @@ func unrollUpExcl(a []int) int { x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { - x += a[i] + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -809,7 +809,7 @@ func unrollUpIncl(a []int) int { x += a[i+1] // ERROR "Proved IsInBounds( for blocked indexing)?$" } if i == len(a)-1 { - x += a[i] + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -822,7 +822,7 @@ func unrollDownExcl0(a []int) int { x += a[i-1] // ERROR "Proved IsInBounds$" } if i == 0 { - x += a[i] + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -835,7 +835,7 @@ func unrollDownExcl1(a []int) int { x += a[i-1] // ERROR "Proved IsInBounds$" } if i == 0 { - x += a[i] + x += a[i] // ERROR "Proved IsInBounds$" } return x } @@ -848,7 +848,7 @@ func unrollDownInclStep(a []int) int { x += a[i-2] // ERROR "Proved IsInBounds$" } if i == 1 { - x += a[i-1] + x += a[i-1] // ERROR "Proved IsInBounds$" } return x } @@ -2896,5 +2896,72 @@ func useInt(a int) { func useSlice(a []int) { } +func testSubSlicingAdd(buf []byte) { + if len(buf) >= 128 { + for i := 0; i <= len(buf)-128; i += 128 { // ERROR "Induction variable:" + _ = buf[i : i+32] // ERROR "Proved IsSliceInBounds" + _ = buf[i+32 : i+64] // ERROR "Proved IsSliceInBounds" + _ = buf[i+64 : i+96] // ERROR "Proved IsSliceInBounds" + _ = buf[i+96 : i+128] // ERROR "Proved IsSliceInBounds" + } + } +} + +func testSubSlicingSub(buf []byte, i int) { + if i >= 128 && i <= len(buf) { + _ = buf[i-128 : i-96] // ERROR "Proved IsSliceInBounds$" + _ = buf[i-96 : i-64] // ERROR "Proved IsSliceInBounds$" + _ = buf[i-64 : i-32] // ERROR "Proved IsSliceInBounds$" + _ = buf[i-32 : i] // ERROR "Proved IsSliceInBounds$" + } +} + +func testSubSlicingAddCanOverflow(buf []byte, i int8) { + if int(i) < len(buf)-10 { + _ = buf[i+1 : i+10] + } +} + +func testSubSlicingSubCanUnderflow(buf []byte, i uint) { + if i <= uint(len(buf)) { + _ = buf[i-64 : i-32] + } +} + +func testConsecutiveLoops(buf []byte) { + i := 0 + n := len(buf) + for ; i <= n-128; i += 128 { // ERROR "Induction variable:" + _ = buf[i : i+32] // ERROR "Proved IsSliceInBounds" + } + for ; i <= n-32; i += 32 { // ERROR "Induction variable:" + _ = buf[i : i+32] // ERROR "Proved IsSliceInBounds" + } +} + +func testDownwardLoopProved(buf []byte) { + if len(buf) >= 15 { + i := 10 + for ; i > 0; i -= 3 { // ERROR "Induction variable:" + _ = buf[i : i+2] // ERROR "Proved IsSliceInBounds" + } + } +} + +func testConsecutiveLoopsMixed(buf []byte) { + i := 10 + n := len(buf) + if n >= 100 { + for ; i > 0; i -= 3 { // ERROR "Induction variable:" + _ = buf[i : i+2] // ERROR "Proved IsSliceInBounds" + } + j := i + for ; j <= n-32; j += 32 { // ERROR "Induction variable:" + // We cannot prove buf[i : i+32] here because i starts negative. + _ = buf[j : j+32] + } + } +} + func main() { }