Skip to content

[pull] master from golang:master #195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 1 commit into from
Sep 2, 2026
Merged

[pull] master from golang:master #195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions src/cmd/compile/internal/arm64/simdssa_sve.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

102 changes: 102 additions & 0 deletions src/cmd/compile/internal/arm64/ssa.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -829,6 +829,14 @@ func ssaGenValue(s *ssagen.State, v *ssa.Value) {
p.From.Offset = v.AuxInt
p.To.Type = obj.TYPE_REG
p.To.Reg = v.Reg()
case ssaop.OpARM64ZSELB:
simdZ2kv(s, v, arm64.ARNG_B)
case ssaop.OpARM64ZSELH:
simdZ2kv(s, v, arm64.ARNG_H)
case ssaop.OpARM64ZSELS:
simdZ2kv(s, v, arm64.ARNG_S)
case ssaop.OpARM64ZSELD:
simdZ2kv(s, v, arm64.ARNG_D)
case ssaop.OpARM64PWHILELTB:
simdPWHILELT(s, v, arm64.ARNG_B)
case ssaop.OpARM64PWHILELTH:
Expand Down Expand Up @@ -2122,6 +2130,100 @@ func simdZ21(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
return p
}

// simdZ2kv emits an SVE instruction that takes a predicate as a plain data
// operand rather than a governing predicate, e.g. ZSEL Z1.B, Z0.B, P0, Z2.B.
// Unlike a predicated instruction it is constructive: the destination is
// independent of the sources, so it needs no MOVPRFX. SSA provides arg0=x (kept
// where the predicate is true), arg1=y (kept where false) and arg2=the predicate.
func simdZ2kv(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = zregArng(v.Args[1].Reg(), arng) // Zm
p.AddRestSourceReg(zregArng(v.Args[0].Reg(), arng)) // Zn
p.AddRestSourceReg(v.Args[2].Reg()) // Pv, unqualified
p.To.Type = obj.TYPE_REG
p.To.Reg = zregArng(v.Reg(), arng) // Zd
return p
}

// simdZ2kvPred emits an SVE predicated binary operation, e.g. ZADD Z1.B, Z0.B, P0.M,
// Z0.B. These instructions are destructive: the destination is also the first
// source. How the destination is put in place depends on what the register
// allocator chose:
//
// - dst == arg0: already destructive-ready, emit the instruction alone.
// - dst == arg1: only reachable for a commutative operation (a
// non-commutative one is marked resultInArg0, which pins dst to arg0), so
// swap the sources and it becomes the case above.
// - anything else: prefix MOVPRFX, which hints the hardware to fuse the pair
// into one constructive operation and leaves both sources intact.
func simdZ2kvPred(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
x, y := v.Args[0].Reg(), v.Args[1].Reg()
d := v.Reg()
switch d {
case x:
case y:
x, y = y, x
default:
mp := s.Prog(arm64.AZMOVPRFX)
mp.From.Type = obj.TYPE_REG
mp.From.Reg = pzreg(x)
mp.To.Type = obj.TYPE_REG
mp.To.Reg = pzreg(d)
}
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = zregArng(y, arng) // Zm
p.AddRestSourceReg(zregArng(d, arng)) // Zdn
p.AddRestSourceReg(pregMask(v.Args[2].Reg(), arm64.PRED_M)) // Pg/M
p.To.Type = obj.TYPE_REG
p.To.Reg = zregArng(d, arng) // Zdn
return p
}

// simdZ3kvPredResultInArg0 emits an SVE merging-predicated binary operation
// whose inactive lanes come from a value that is neither of its sources, e.g.
// x.Add(y).IfElse(mask, z). SSA provides arg0=z, arg1=x, arg2=y, arg3=mask, and
// resultInArg0 puts z in the destination register.
//
// ZMOVPRFX Zx, Pg/M, Zd // Zd = x on the active lanes, z on the rest
// ZADD Zy, Zd, Pg/M, Zd // Zd = x+y on the active lanes, z on the rest
//
// The prefix must be the predicated MOVPRFX, not the unpredicated one: the
// whole-register form would leave x rather than z in the inactive lanes.
//
// A prefixed instruction may not name its destination in any operand position
// other than the destructive one, so Zd must differ from the operation's Zm.
// Only a commutative operation gets this lowering (see sveMergingPrefixedOp in
// simdgen), so when the register allocator puts the destination on one source —
// which it can only do when z is that source — the other one becomes Zm and the
// prefix is unnecessary.
func simdZ3kvPredResultInArg0(s *ssagen.State, v *ssa.Value, arng int16) *obj.Prog {
d := v.Reg()
x, y := v.Args[1].Reg(), v.Args[2].Reg()
pg := v.Args[3].Reg()
zn, zm := x, y
if d == zm {
zn, zm = zm, zn
}
if d != zn {
mp := s.Prog(arm64.AZMOVPRFX)
mp.From.Type = obj.TYPE_REG
mp.From.Reg = zregArng(zn, arng)
mp.AddRestSourceReg(pregMask(pg, arm64.PRED_M))
mp.To.Type = obj.TYPE_REG
mp.To.Reg = zregArng(d, arng)
}
p := s.Prog(v.Op.Asm())
p.From.Type = obj.TYPE_REG
p.From.Reg = zregArng(zm, arng) // Zm
p.AddRestSourceReg(zregArng(d, arng)) // Zdn
p.AddRestSourceReg(pregMask(pg, arm64.PRED_M)) // Pg/M
p.To.Type = obj.TYPE_REG
p.To.Reg = zregArng(d, arng) // Zdn
return p
}

// simdPWHILELT emits a PWHILELT that fills a predicate with lanes [lo,hi) set for
// the given element arrangement, e.g. PWHILELT R0, R1, P0.B. SSA provides
// arg0=lo, arg1=hi.
Expand Down
13 changes: 13 additions & 0 deletions src/cmd/compile/internal/ssa/_gen/ARM64.rules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1905,5 +1905,18 @@
// These are written by hand until the predicated ops are generated by simdgen
// in the mask CL; at that point delete them and generate instead.
(Count8s r) => (Select0 <types.TypeMask> (PWHILELTB (MOVDconst [0]) r))

// SVE per-element select, backing IfElse and Masked.
(IfElseInt8s x mask y) => (ZSELB x y mask)
(IfElseUint8s x mask y) => (ZSELB x y mask)
(IfElseInt16s x mask y) => (ZSELH x y mask)
(IfElseUint16s x mask y) => (ZSELH x y mask)
(IfElseInt32s x mask y) => (ZSELS x y mask)
(IfElseUint32s x mask y) => (ZSELS x y mask)
(IfElseFloat32s x mask y) => (ZSELS x y mask)
(IfElseInt64s x mask y) => (ZSELD x y mask)
(IfElseUint64s x mask y) => (ZSELD x y mask)
(IfElseFloat64s x mask y) => (ZSELD x y mask)

(LoadMasked8 <t> ptr mask mem) && t.Size() == 32 => (ZLD1BPredload ptr mask mem)
(StoreMasked8 {t} ptr mask val mem) && t.Size() == 32 => (ZST1BPredstore ptr val mask mem)
8 changes: 7 additions & 1 deletion src/cmd/compile/internal/ssa/_gen/ARM64Ops.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ func init() {
fp2flags = regInfo{inputs: []regMask{fp, fp}}
fp1flags = regInfo{inputs: []regMask{fp}}
fp2predpred = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{pred}}
fp2predfp = regInfo{inputs: []regMask{fp, fp, pred}, outputs: []regMask{fp}}
fp3predfp = regInfo{inputs: []regMask{fp, fp, fp, pred}, outputs: []regMask{fp}}
predload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{pred}}
predstore = regInfo{inputs: []regMask{gpspsbg, pred}}
fpload = regInfo{inputs: []regMask{gpspsbg}, outputs: []regMask{fp}}
Expand Down Expand Up @@ -839,6 +841,10 @@ func init() {
// scalable Z bank reuses the fp register masks.
{name: "ZLDRload", argLength: 2, reg: fpload, aux: "SymOff", asm: "ZLDR", typ: "Vec256", faultOnNilArg0: true, symEffect: "Read"}, // load from arg0 + auxInt + aux. arg1=mem.
{name: "ZSTRstore", argLength: 3, reg: fpstore, aux: "SymOff", asm: "ZSTR", faultOnNilArg0: true, symEffect: "Write"}, // store arg1 to arg0 + auxInt + aux. arg2=mem.
{name: "ZSELB", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive.
{name: "ZSELH", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive.
{name: "ZSELS", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive.
{name: "ZSELD", argLength: 3, reg: fp2predfp, asm: "ZSEL", typ: "Vec256"}, // arg0=x, arg1=y, arg2=predicate; per-element select, constructive.
{name: "PLDRload", argLength: 2, reg: predload, aux: "SymOff", asm: "PLDR", typ: "Mask", faultOnNilArg0: true, symEffect: "Read"}, // load a predicate from arg0 + auxInt + aux. arg1=mem.
{name: "PSTRstore", argLength: 3, reg: predstore, aux: "SymOff", asm: "PSTR", faultOnNilArg0: true, symEffect: "Write"}, // store predicate arg1 to arg0 + auxInt + aux. arg2=mem.
// PPFALSEB sets every bit of a predicate false, it's the zero value of a predicate.
Expand Down Expand Up @@ -896,7 +902,7 @@ func init() {
pkg: "cmd/internal/obj/arm64",
genfile: "../../arm64/ssa.go",
genSIMDfile: "../../arm64/simdssa.go ../../arm64/simdssa_sve.go",
ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred)...),
ops: append(append(ops, simdARM64Ops(fp11, fp21, fp31, fpgp, fpgpfp, fp21)...), simdARM64SVEOps(fp11, fp21, fp2predpred, fp2predfp, fp2predfp, fp3predfp)...),
blocks: blocks,
regnames: regNamesARM64,
ParamIntRegNames: "R0 R1 R2 R3 R4 R5 R6 R7 R8 R9 R10 R11 R12 R13 R14 R15",
Expand Down
16 changes: 16 additions & 0 deletions src/cmd/compile/internal/ssa/_gen/genericOps.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,22 @@ var genericOps = []opData{

{name: "ScalableVectorLen", argLength: 0}, // SVE runtime vector length in bytes
{name: "Count8s", argLength: 1}, // arg0 = active byte count; builds an SVE predicate over that many byte lanes

// IfElse selects per element between two scalable vectors under a predicate.
// It backs both the IfElse and (against a zero vector) the Masked method, and
// is written by hand rather than derived from the ISA because SEL is
// bit-pattern-agnostic: there is no float-lane encoding of it to unify with.
// arg0 = x, arg1 = predicate, arg2 = y (taken where the predicate is false).
{name: "IfElseInt8s", argLength: 3},
{name: "IfElseUint8s", argLength: 3},
{name: "IfElseInt16s", argLength: 3},
{name: "IfElseUint16s", argLength: 3},
{name: "IfElseInt32s", argLength: 3},
{name: "IfElseUint32s", argLength: 3},
{name: "IfElseFloat32s", argLength: 3},
{name: "IfElseInt64s", argLength: 3},
{name: "IfElseUint64s", argLength: 3},
{name: "IfElseFloat64s", argLength: 3},
}

// kind controls successors implicit exit
Expand Down
42 changes: 42 additions & 0 deletions src/cmd/compile/internal/ssa/_gen/simdARM64SVE.rules
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,3 +22,45 @@
(GreaterInt32s x y) => (ZCMPGTS x y (Select0 <types.TypeMask> (PWHILELTS (MOVDconst [0]) (MOVDconst [8]))))
(GreaterInt64s x y) => (ZCMPGTD x y (Select0 <types.TypeMask> (PWHILELTD (MOVDconst [0]) (MOVDconst [4]))))
(GreaterInt8s x y) => (ZCMPGTB x y (Select0 <types.TypeMask> (PWHILELTB (MOVDconst [0]) (MOVDconst [32]))))
(ZSELB (ZADDB x y) x mask) => (ZADDMergingB x y mask)
(ZSELB (ZADDB x y) y mask) => (ZADDMergingB y x mask)
(ZSELB (ZADDB x y) z mask) => (ZADDMergingPrefixedB z x y mask)
(ZSELB (ZSQADDB x y) x mask) => (ZSQADDMergingB x y mask)
(ZSELB (ZSQADDB x y) y mask) => (ZSQADDMergingB y x mask)
(ZSELB (ZSQADDB x y) z mask) => (ZSQADDMergingPrefixedB z x y mask)
(ZSELB (ZUQADDB x y) x mask) => (ZUQADDMergingB x y mask)
(ZSELB (ZUQADDB x y) y mask) => (ZUQADDMergingB y x mask)
(ZSELB (ZUQADDB x y) z mask) => (ZUQADDMergingPrefixedB z x y mask)
(ZSELD (ZADDD x y) x mask) => (ZADDMergingD x y mask)
(ZSELD (ZADDD x y) y mask) => (ZADDMergingD y x mask)
(ZSELD (ZADDD x y) z mask) => (ZADDMergingPrefixedD z x y mask)
(ZSELD (ZFADDD x y) x mask) => (ZFADDMergingD x y mask)
(ZSELD (ZFADDD x y) y mask) => (ZFADDMergingD y x mask)
(ZSELD (ZFADDD x y) z mask) => (ZFADDMergingPrefixedD z x y mask)
(ZSELD (ZSQADDD x y) x mask) => (ZSQADDMergingD x y mask)
(ZSELD (ZSQADDD x y) y mask) => (ZSQADDMergingD y x mask)
(ZSELD (ZSQADDD x y) z mask) => (ZSQADDMergingPrefixedD z x y mask)
(ZSELD (ZUQADDD x y) x mask) => (ZUQADDMergingD x y mask)
(ZSELD (ZUQADDD x y) y mask) => (ZUQADDMergingD y x mask)
(ZSELD (ZUQADDD x y) z mask) => (ZUQADDMergingPrefixedD z x y mask)
(ZSELH (ZADDH x y) x mask) => (ZADDMergingH x y mask)
(ZSELH (ZADDH x y) y mask) => (ZADDMergingH y x mask)
(ZSELH (ZADDH x y) z mask) => (ZADDMergingPrefixedH z x y mask)
(ZSELH (ZSQADDH x y) x mask) => (ZSQADDMergingH x y mask)
(ZSELH (ZSQADDH x y) y mask) => (ZSQADDMergingH y x mask)
(ZSELH (ZSQADDH x y) z mask) => (ZSQADDMergingPrefixedH z x y mask)
(ZSELH (ZUQADDH x y) x mask) => (ZUQADDMergingH x y mask)
(ZSELH (ZUQADDH x y) y mask) => (ZUQADDMergingH y x mask)
(ZSELH (ZUQADDH x y) z mask) => (ZUQADDMergingPrefixedH z x y mask)
(ZSELS (ZADDS x y) x mask) => (ZADDMergingS x y mask)
(ZSELS (ZADDS x y) y mask) => (ZADDMergingS y x mask)
(ZSELS (ZADDS x y) z mask) => (ZADDMergingPrefixedS z x y mask)
(ZSELS (ZFADDS x y) x mask) => (ZFADDMergingS x y mask)
(ZSELS (ZFADDS x y) y mask) => (ZFADDMergingS y x mask)
(ZSELS (ZFADDS x y) z mask) => (ZFADDMergingPrefixedS z x y mask)
(ZSELS (ZSQADDS x y) x mask) => (ZSQADDMergingS x y mask)
(ZSELS (ZSQADDS x y) y mask) => (ZSQADDMergingS y x mask)
(ZSELS (ZSQADDS x y) z mask) => (ZSQADDMergingPrefixedS z x y mask)
(ZSELS (ZUQADDS x y) x mask) => (ZUQADDMergingS x y mask)
(ZSELS (ZUQADDS x y) y mask) => (ZUQADDMergingS y x mask)
(ZSELS (ZUQADDS x y) z mask) => (ZUQADDMergingPrefixedS z x y mask)
30 changes: 29 additions & 1 deletion src/cmd/compile/internal/ssa/_gen/simdARM64SVEops.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading
Loading