From a97b136d6697030259345f9a6423e28dd7caed9d Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Sat, 1 Aug 2026 15:09:09 -0500 Subject: [PATCH 01/21] fix a linter issue --- hash/hash_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hash/hash_test.go b/hash/hash_test.go index 09de08d8..d1b49fff 100644 --- a/hash/hash_test.go +++ b/hash/hash_test.go @@ -21,13 +21,14 @@ package hash import ( "crypto/rand" "crypto/sha256" + "crypto/sha3" "crypto/sha512" "encoding/hex" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "golang.org/x/crypto/sha3" + xsha3 "golang.org/x/crypto/sha3" ) // Sanity check of SHA3_256 @@ -260,7 +261,7 @@ func TestKeccak(t *testing.T) { value := make([]byte, i) _, err := rand.Read(value) require.NoError(t, err) - k := sha3.NewLegacyKeccak256() + k := xsha3.NewLegacyKeccak256() k.Write(value) expected := k.Sum(nil) From e54f62f8b7900afe8fdb217174b9c92fc46ed0a2 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Sat, 1 Aug 2026 15:11:47 -0500 Subject: [PATCH 02/21] refactor ecdsa and use go-ethereum for secp256k1 operations --- bls12381_utils.go | 4 + ecdsa.go | 508 ++++++++++++--------------------------------- ecdsa_p256.go | 275 ++++++++++++++++++++++++ ecdsa_secp256k1.go | 267 ++++++++++++++++++++++++ ecdsa_test.go | 51 +---- go.mod | 8 +- go.sum | 25 ++- sign.go | 15 +- 8 files changed, 696 insertions(+), 457 deletions(-) create mode 100644 ecdsa_p256.go create mode 100644 ecdsa_secp256k1.go diff --git a/bls12381_utils.go b/bls12381_utils.go index e504a914..aa939c79 100644 --- a/bls12381_utils.go +++ b/bls12381_utils.go @@ -117,6 +117,10 @@ func initBLS12381() { // set a global point to infinity C.E2_set_infty((*C.E2)(&g2PublicKey.point)) g2PublicKey.isIdentity = true + + blsInstance = &blsBLS12381Algo{ + algo: BLSBLS12381, + } } // String returns a hex-encoded representation of the scalar. diff --git a/ecdsa.go b/ecdsa.go index 6a2ae135..9e4e715b 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -21,168 +21,70 @@ package crypto // Elliptic Curve Digital Signature Algorithm is implemented as // defined in FIPS 186-4 (although the hash functions implemented in this package are SHA2 and SHA3). -// Most of the implementation is Go based and is not optimized for performance. - -// This implementation does not include any security against side-channel attacks. +// This implementation is not resistant against side-channel attacks or fault attacks. import ( - "crypto/ecdh" - "crypto/ecdsa" - "crypto/elliptic" + "bytes" "crypto/hkdf" - "crypto/rand" "crypto/sha256" "fmt" "math/big" - "github.com/btcsuite/btcd/btcec/v2" - "github.com/onflow/crypto/hash" ) -const ( - // NIST P256 - SignatureLenECDSAP256 = 64 - PrKeyLenECDSAP256 = 32 - // PubKeyLenECDSAP256 is the size of uncompressed points on P256 - PubKeyLenECDSAP256 = 64 - - // SECG secp256k1 - SignatureLenECDSASecp256k1 = 64 - PrKeyLenECDSASecp256k1 = 32 - // PubKeyLenECDSASecp256k1 is the size of uncompressed points on secp256k1 - PubKeyLenECDSASecp256k1 = 64 -) - -// ecdsaAlgo embeds SignAlgo -type ecdsaAlgo struct { - // elliptic curve - curve elliptic.Curve - // the signing algo and parameters +// ecdsaContext embeds SigningAlgorithm +type ecdsaContext struct { + // the signing algo algo SigningAlgorithm + // curve prime field + curveP *big.Int + // curve order + curveN *big.Int } -// ECDSA contexts for each supported curve -// -// NIST P-256 curve -var p256Instance *ecdsaAlgo +const ecEncodingUncompressed = 0x4 -// SECG secp256k1 curve https://www.secg.org/sec2-v2.pdf -var secp256k1Instance *ecdsaAlgo +func initECDSA() { + // ECDSA with P256 + initECDSAP256() + // ECDSA with secp256k1 + initECDSASecp256k1() +} func bitsToBytes(bits int) int { return (bits + 7) >> 3 } -// signHash returns the signature of the input hash using the private key receiver. -// The signature is the concatenation bytes(r) || bytes(s), -// where `r` and `s` are padded to the curve order size. -// Current implementation of `sign` is randomized, mixing the entropy from the -// the system's crypto/rand, the private key and the hash. -// -// The caller must make sure that the hash is at least the curve order size. -func (sk *prKeyECDSA) signHash(h hash.Hash) (Signature, error) { - r, s, err := ecdsa.Sign(rand.Reader, sk.goPrKey, h) - if err != nil { - return nil, fmt.Errorf("ECDSA sign failed: %w", err) - } - rBytes := r.Bytes() - sBytes := s.Bytes() - nLen := bitsToBytes((sk.alg.curve.Params().N).BitLen()) - signature := make([]byte, 2*nLen) - // pad the signature with zeroes - copy(signature[nLen-len(rBytes):], rBytes) - copy(signature[2*nLen-len(sBytes):], sBytes) - return signature, nil -} - -// Sign signs an array of bytes -// -// The resulting signature is the concatenation bytes(r)||bytes(s), -// where r and s are padded to the curve order size. -// The private key is read only while sha2 and sha3 hashers are -// modified temporarily. -// -// The function returns: -// - (false, errNilHasher) if a hasher is nil -// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). -// - (nil, error) if an unexpected error occurs -// - (signature, nil) otherwise -func (sk *prKeyECDSA) Sign(data []byte, alg hash.Hasher) (Signature, error) { - if alg == nil { +func (a *ecdsaContext) checkAlgoAndComputeHash(msg []byte, hasher hash.Hasher) (hash.Hash, error) { + if hasher == nil { return nil, errNilHasher } - // check hasher's size is at least the curve order in bytes - nLen := bitsToBytes((sk.alg.curve.Params().N).BitLen()) - if alg.Size() < nLen { - return nil, invalidHasherSizeErrorf( - "hasher's size should be at least %d, got %d", nLen, alg.Size()) - } - - h := alg.ComputeHash(data) - return sk.signHash(h) -} - -// verifyHash implements ECDSA signature verification -func (pk *pubKeyECDSA) verifyHash(sig Signature, h hash.Hash) (bool, error) { - nLen := bitsToBytes((pk.alg.curve.Params().N).BitLen()) - - if len(sig) != 2*nLen { - return false, nil - } - - var r big.Int - var s big.Int - r.SetBytes(sig[:nLen]) - s.SetBytes(sig[nLen:]) - return ecdsa.Verify(pk.goPubKey, h, &r, &s), nil -} - -// Verify verifies a signature of an input data under the public key. -// -// If the input signature slice has an invalid length or fails to deserialize into valid -// scalars, the function returns false without an error. -// -// Public keys are read only, sha2 and sha3 hashers are -// modified temporarily. -// -// The function returns: -// - (false, errNilHasher) if a hasher is nil -// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). -// - (false, error) if an unexpected error occurs -// - (validity, nil) otherwise -func (pk *pubKeyECDSA) Verify(sig Signature, data []byte, alg hash.Hasher) (bool, error) { - if alg == nil { - return false, errNilHasher - } // check hasher's size is at least the curve order in bytes - nLen := bitsToBytes((pk.alg.curve.Params().N).BitLen()) - if alg.Size() < nLen { - return false, invalidHasherSizeErrorf( - "hasher's size should be at least %d, got %d", nLen, alg.Size()) + nLen := bitsToBytes((a.curveN).BitLen()) + if hasher.Size() < nLen { + return nil, invalidHasherSizeErrorf( + "hasher's size should be at least %d, got %d", nLen, hasher.Size()) } - h := alg.ComputeHash(data) - return pk.verifyHash(sig, h) + h := hasher.ComputeHash(msg) + return h, nil } // signatureFormatCheck verifies the format of a serialized signature, // regardless of messages or public keys. // If FormatCheck returns false then the input is not a valid ECDSA // signature and will fail a verification against any message and public key. -func (a *ecdsaAlgo) signatureFormatCheck(sig Signature) bool { - N := a.curve.Params().N +func (a *ecdsaContext) signatureFormatCheck(sig Signature) bool { + N := a.curveN nLen := bitsToBytes(N.BitLen()) if len(sig) != 2*nLen { return false } - var r big.Int - var s big.Int - r.SetBytes(sig[:nLen]) - s.SetBytes(sig[nLen:]) + r, s := readTwoBigInts(sig, nLen) if r.Sign() == 0 || s.Sign() == 0 { return false @@ -199,61 +101,43 @@ func (a *ecdsaAlgo) signatureFormatCheck(sig Signature) bool { var one = new(big.Int).SetInt64(1) -// goecdsaMapKey maps the input seed to a private key -// of the Go crypto/ecdsa library. +// mapToPrivateKey simply maps the input seed to an ECDSA private key // The private scalar `d` satisfies 0 < d < n. -// Returned error is expected to be nil. -func goecdsaMapKey(curve elliptic.Curve, seed []byte) (*ecdsa.PrivateKey, error) { +// +// The function returns: +// - (nil, invalidInputsError) if the curve is not supported +// - (nil, error) if an unexpected error occurs +// - (sk, nil) if key mapping was successful +func (a *ecdsaContext) mapToPrivateKey(seed []byte) (PrivateKey, error) { d := new(big.Int).SetBytes(seed) - n := new(big.Int).Sub(curve.Params().N, one) - d.Mod(d, n) - d.Add(d, one) - return goecdsaPrivateKey(curve, d) // n > d > 0 at this point + NminusOne := new(big.Int).Sub(a.curveN, one) + d.Mod(d, NminusOne) + d.Add(d, one) // n > d > 0 at this point + return a.privateKey(d) } -// goecdsaPrivateKey creates a Go crypto/ecdsa private key using the -// input curve and scalar. -// Input scalar is assumed to be a non-zero integer modulo the curve order `n`. -// Error returns: -// - invalidInputsError if the input curve is unsupported -func goecdsaPrivateKey(curve elliptic.Curve, d *big.Int) (*ecdsa.PrivateKey, error) { - priv := new(ecdsa.PrivateKey) - priv.D = d - priv.PublicKey.Curve = curve - - // compute the crypto/ecdsa public key - if curve == elliptic.P256() { - // Perform the base scalar multiplication using crypto/ecdh, - // because crypto/elliptic deprecated `ScalarBaseMult`. - // - // We build the ecdh.PrivateKey directly from the scalar bytes - // instead of going through `priv.ECDH()`: since Go 1.26, - // ecdsa's `(*PrivateKey).ECDH` serializes the key via `(*PrivateKey).Bytes`, - // which reads the public affine coordinates `X`/`Y`. - // Those are not set yet at this point (we are computing them), - // so that path dereferences nil and panics. - // Constructing the ecdh key from the scalar avoids reading `X`/`Y` - // and works across Go versions. - scalarLen := bitsToBytes(curve.Params().N.BitLen()) - ecdhPriv, err := ecdh.P256().NewPrivateKey(d.FillBytes(make([]byte, scalarLen))) - if err != nil { - // at this point, no error is expected because the function can't be called - // with a zero scalar modulo `n` - return nil, fmt.Errorf("non expected error when creating an ECDH private key: %w", err) - } - // crypto/ecdh serialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3). - // The bytes returned are `0x04 || X || Y` because the point is guaranteed to be non-infinity - ecdhPubBytes := ecdhPriv.PublicKey().Bytes() - pLen := bitsToBytes(curve.Params().P.BitLen()) - priv.PublicKey.X = new(big.Int).SetBytes(ecdhPubBytes[1 : 1+pLen]) - priv.PublicKey.Y = new(big.Int).SetBytes(ecdhPubBytes[1+pLen:]) - } else if curve == btcec.S256() { - // `ScalarBaseMult` is not deprecated in btcec's type `KoblitzCurve` - priv.PublicKey.X, priv.PublicKey.Y = btcec.S256().ScalarBaseMult(d.Bytes()) - } else { +// privateKey returns an ECDSA private key using the +// input scalar. + +// Input scalar d is assumed to be satisfy 0 < d < n before calling this function. +// +// The function returns: +// - (nil, invalidInputsError) if the curve is not supported +// - (nil, error) if an unexpected error occurs +// - (sk, nil) if key mapping was successful +func (a *ecdsaContext) privateKey(d *big.Int) (PrivateKey, error) { + dBytes := make([]byte, bitsToBytes(a.curveN.BitLen())) + d.FillBytes(dBytes) // dBytes is the big-endian encoding of d padded to the curve order + + // build the private key depending on the curve + switch a.algo { + case ECDSAP256: + return privateKeyECDSAP256(a, dBytes) + case ECDSASecp256k1: + return privateKeyECDSASecp256k1(a, dBytes), nil + default: return nil, invalidInputsErrorf("the curve is not supported") } - return priv, nil } // generatePrivateKey generates a private key for ECDSA @@ -261,7 +145,7 @@ func goecdsaPrivateKey(curve elliptic.Curve, d *big.Int) (*ecdsa.PrivateKey, err // // It is recommended to use a secure crypto RNG to generate the seed. // The seed must have enough entropy. -func (a *ecdsaAlgo) generatePrivateKey(seed []byte) (PrivateKey, error) { +func (a *ecdsaContext) generatePrivateKey(seed []byte) (PrivateKey, error) { if len(seed) < KeyGenSeedMinLen || len(seed) > KeyGenSeedMaxLen { return nil, invalidInputsErrorf("seed byte length should be between %d and %d", KeyGenSeedMinLen, KeyGenSeedMaxLen) @@ -274,7 +158,7 @@ func (a *ecdsaAlgo) generatePrivateKey(seed []byte) (PrivateKey, error) { salt := []byte("") // HKDF salt info := "" // HKDF info // use extra 128 bits to reduce the modular reduction bias - nLen := bitsToBytes((a.curve.Params().N).BitLen()) + nLen := bitsToBytes((a.curveN).BitLen()) okmLength := nLen + (securityBits / 8) // instantiate HKDF and extract okm @@ -284,23 +168,19 @@ func (a *ecdsaAlgo) generatePrivateKey(seed []byte) (PrivateKey, error) { } defer overwrite(okm) // overwrite okm - sk, err := goecdsaMapKey(a.curve, okm) + sk, err := a.mapToPrivateKey(okm) if err != nil { // no error is expected at this point return nil, fmt.Errorf("mapping the private key failed: %w", err) } - return &prKeyECDSA{ - alg: a, - goPrKey: sk, - pubKey: nil, // public key is not constructed - }, nil + return sk, nil } -func (a *ecdsaAlgo) rawDecodePrivateKey(der []byte) (PrivateKey, error) { - n := a.curve.Params().N +func (a *ecdsaContext) rawDecodePrivateKey(der []byte) (PrivateKey, error) { + n := a.curveN nLen := bitsToBytes(n.BitLen()) if len(der) != nLen { - return nil, invalidInputsErrorf("input has incorrect %s key size", a.algo) + return nil, invalidInputsErrorf("input has incorrect %s key size, should be %d", a.algo, nLen) } var d big.Int d.SetBytes(der) @@ -313,20 +193,16 @@ func (a *ecdsaAlgo) rawDecodePrivateKey(der []byte) (PrivateKey, error) { return nil, invalidInputsErrorf("zero private keys are not a valid %s key", a.algo) } - priv, err := goecdsaPrivateKey(a.curve, &d) // n > d > 0 at this point + sk, err := a.privateKey(&d) // n > d > 0 at this point if err != nil { // error is not expected at this point return nil, fmt.Errorf("building the private key failed: %w", err) } - return &prKeyECDSA{ - alg: a, - goPrKey: priv, - pubKey: nil, // public key is not constructed - }, nil + return sk, nil } -func (a *ecdsaAlgo) decodePrivateKey(der []byte) (PrivateKey, error) { +func (a *ecdsaContext) decodePrivateKey(der []byte) (PrivateKey, error) { return a.rawDecodePrivateKey(der) } @@ -335,236 +211,112 @@ func (a *ecdsaAlgo) decodePrivateKey(der []byte) (PrivateKey, error) { // Note that infinity point serialization isn't defined in this package so the input (or output) can never represent an infinity point. // Error Returns: // - invalidInputsError if the input is not a valid serialization of a public key on the given curve. -func (a *ecdsaAlgo) rawDecodePublicKey(der []byte) (PublicKey, error) { - curve := a.curve - p := (curve.Params().P) - pLen := bitsToBytes(p.BitLen()) - if len(der) != 2*pLen { - return nil, invalidInputsErrorf("input has incorrect %s key size, got %d, expects %d", - a.algo, len(der), 2*pLen) - } - var x, y big.Int - x.SetBytes(der[:pLen]) - y.SetBytes(der[pLen:]) - - // check the coordinates are valid field elements - if x.Cmp(p) >= 0 || y.Cmp(p) >= 0 { - return nil, invalidInputsErrorf("at least one coordinate is larger than the field prime for %s", a.algo) - } - +func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { // all the curves supported for now have a cofactor equal to 1, - // so that checking the point is on curve is enough. - if curve == elliptic.P256() { - // use crypto/ecdh implementation to perform on curve check - // because crypto/elliptic deprecated `IsOnCurve`. - // ECDH's `NewPublicKey` checks the public key is on curve to avoid falling in small-order groups. - - // crypto/ecdh deserialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3) - // except for infinity point. - // The bytes serialization for non-zero points is `0x04 || X || Y` - ecdhPubBytes := append([]byte{0x4}, der...) - - _, err := ecdh.P256().NewPublicKey(ecdhPubBytes) - if err != nil { - return nil, invalidInputsErrorf("input is not a point on curve P-256: %w", err) - } - } else if curve == btcec.S256() { - // `IsOnCurve` is not deprecated in btcec's type `KoblitzCurve` - if !btcec.S256().IsOnCurve(&x, &y) { - return nil, invalidInputsErrorf("input is not a point on curve secp256k1") - } - } else { + // so that checking the point is on curve is enough to make sure it is on the correct subgroup + switch a.algo { + case ECDSAP256: + return publicKeyECDSAP256(a, input) + case ECDSASecp256k1: + return publicKeyECDSASecp256k1(a, input) + default: return nil, invalidInputsErrorf("curve is not supported") } - pk := ecdsa.PublicKey{ - Curve: a.curve, - X: &x, - Y: &y, - } - - return &pubKeyECDSA{a, &pk}, nil } -func (a *ecdsaAlgo) decodePublicKey(der []byte) (PublicKey, error) { +func (a *ecdsaContext) decodePublicKey(der []byte) (PublicKey, error) { return a.rawDecodePublicKey(der) } // decodePublicKeyCompressed returns a non-infinity public key given the bytes of a compressed // public key according to X9.62 section 4.3.6. -// The compressed representation uses an extra byte to disambiguate sign. // Note that infinity point serialization isn't defined in this package so the input (or output) // can never represent an infinity point. // Error Returns: // - invalidInputsError if the curve isn't supported or the input isn't a valid key serialization // on the given curve. -func (a *ecdsaAlgo) decodePublicKeyCompressed(pkBytes []byte) (PublicKey, error) { - expectedLen := bitsToBytes(a.curve.Params().BitSize) + 1 - if len(pkBytes) != expectedLen { - return nil, invalidInputsErrorf("input length incompatible, expected %d, got %d", expectedLen, len(pkBytes)) - } - var goPubKey *ecdsa.PublicKey - - if a.curve == elliptic.P256() { - x, y := elliptic.UnmarshalCompressed(a.curve, pkBytes) - if x == nil { - return nil, invalidInputsErrorf("input %x isn't a compressed serialization of a %v key", pkBytes, a.algo.String()) - } - goPubKey = new(ecdsa.PublicKey) - goPubKey.Curve = a.curve - goPubKey.X = x - goPubKey.Y = y - - } else if a.curve == btcec.S256() { - // use `btcec` because elliptic's `UnmarshalCompressed` doesn't work for SEC Koblitz curves - pk, err := btcec.ParsePubKey(pkBytes) - if err != nil { - return nil, invalidInputsErrorf("input %x isn't a compressed serialization of a %v key", pkBytes, a.algo.String()) - } - // convert to a crypto/ecdsa key - goPubKey = pk.ToECDSA() - } else { +func (a *ecdsaContext) decodePublicKeyCompressed(pkBytes []byte) (PublicKey, error) { + switch a.algo { + case ECDSAP256: + return p256DecodePublicKeyCompressed(pkBytes) + case ECDSASecp256k1: + return secp256k1DecodePublicKeyCompressed(pkBytes) + default: return nil, invalidInputsErrorf("the input curve is not supported") } - return &pubKeyECDSA{a, goPubKey}, nil } -// prKeyECDSA is the private key of ECDSA, it implements the interface PrivateKey -type prKeyECDSA struct { - // the signature algo - alg *ecdsaAlgo - // ecdsa private key - goPrKey *ecdsa.PrivateKey - // public key - pubKey *pubKeyECDSA -} - -var _ PrivateKey = (*prKeyECDSA)(nil) - // Algorithm returns the algo related to the private key -func (sk *prKeyECDSA) Algorithm() SigningAlgorithm { - return sk.alg.algo +func (a *ecdsaContext) Algorithm() SigningAlgorithm { + return a.algo } -// Size returns the length of the private key in bytes -func (sk *prKeyECDSA) Size() int { - return bitsToBytes((sk.alg.curve.Params().N).BitLen()) +type prKeyCommonECDSA struct { + // ECDSA context + *ecdsaContext } -// PublicKey returns the public key associated to the private key -func (sk *prKeyECDSA) PublicKey() PublicKey { - // construct the public key once - if sk.pubKey == nil { - sk.pubKey = &pubKeyECDSA{ - alg: sk.alg, - goPubKey: &sk.goPrKey.PublicKey, - } - } - return sk.pubKey +// Size returns the length of the private key in bytes +func (sk *prKeyCommonECDSA) Size() int { + return bitsToBytes((sk.curveN).BitLen()) } -// given a private key (d), returns a raw encoding bytes(d) in big endian -// padded to the private key length -func (sk *prKeyECDSA) rawEncode() []byte { - skBytes := sk.goPrKey.D.Bytes() - nLen := bitsToBytes((sk.alg.curve.Params().N).BitLen()) - skEncoded := make([]byte, nLen) - // pad sk with zeroes - copy(skEncoded[nLen-len(skBytes):], skBytes) - return skEncoded +// prKeyCommonECDSAString returns the string representation of an ECDSA private key. +// It is used by all ECDSA private keys regardless of the curve. +func prKeyCommonECDSAString(sk PrivateKey) string { + return fmt.Sprintf("%#x", sk.Encode()) } -// Encode returns a byte representation of a private key. -// a simple raw byte encoding in big endian is used for all curves -func (sk *prKeyECDSA) Encode() []byte { - return sk.rawEncode() +// pubKeyCommonECDSAString returns the string representation of an ECDSA public key. +// It is used by all ECDSA public keys regardless of the curve. +func pubKeyCommonECDSAString(pk PublicKey) string { + return fmt.Sprintf("%#x", pk.Encode()) } // Equals test the equality of two private keys -func (sk *prKeyECDSA) Equals(other PrivateKey) bool { - // check the key type - otherECDSA, ok := other.(*prKeyECDSA) - if !ok { - return false - } - // check the curve - if sk.alg.curve != otherECDSA.alg.curve { +func prKeyCommonECDSAEquals(sk, other PrivateKey) bool { + // check the algorithm + if sk.Algorithm() != other.Algorithm() { return false } - return sk.goPrKey.D.Cmp(otherECDSA.goPrKey.D) == 0 + // check the scalar + return bytes.Equal(sk.Encode(), other.Encode()) } -// String returns the hex string representation of the key. -func (sk *prKeyECDSA) String() string { - return fmt.Sprintf("%#x", sk.Encode()) -} - -// pubKeyECDSA is the public key of ECDSA, it implements PublicKey -type pubKeyECDSA struct { - // the signature algo - alg *ecdsaAlgo - // public key data - goPubKey *ecdsa.PublicKey -} - -var _ PublicKey = (*pubKeyECDSA)(nil) - -// Algorithm returns the the algo related to the private key -func (pk *pubKeyECDSA) Algorithm() SigningAlgorithm { - return pk.alg.algo +type pubKeyCommonECDSA struct { + // ECDSA context + *ecdsaContext } // Size returns the length of the public key in bytes -func (pk *pubKeyECDSA) Size() int { - return 2 * bitsToBytes((pk.goPubKey.Params().P).BitLen()) -} - -// EncodeCompressed returns a compressed encoding according to X9.62 section 4.3.6. -// This compressed representation uses an extra byte to disambiguate parity. -// The expected input is a public key (x,y). -// -// Receiver point is guaranteed to be on curve and to be non-infinity because -// the package does not allow constructing infinity points or points not on curve. -func (pk *pubKeyECDSA) EncodeCompressed() []byte { - return elliptic.MarshalCompressed(pk.goPubKey.Curve, pk.goPubKey.X, pk.goPubKey.Y) -} - -// `rawEncode` returns a raw uncompressed encoding `bytes(x) || bytes(y)` given a public key (x,y). -// x and y are padded to the field size. -func (pk *pubKeyECDSA) rawEncode() []byte { - xBytes := pk.goPubKey.X.Bytes() - yBytes := pk.goPubKey.Y.Bytes() - Plen := bitsToBytes((pk.alg.curve.Params().P).BitLen()) - pkEncoded := make([]byte, 2*Plen) - // pad the public key coordinates with zeroes - copy(pkEncoded[Plen-len(xBytes):], xBytes) - copy(pkEncoded[2*Plen-len(yBytes):], yBytes) - return pkEncoded -} - -// Encode returns a byte representation of a public key. -// a simple uncompressed raw encoding X||Y is used for all curves -// X and Y are the big endian byte encoding of the x and y coordinates of the public key -func (pk *pubKeyECDSA) Encode() []byte { - return pk.rawEncode() +func (pk *pubKeyCommonECDSA) Size() int { + return 2 * bitsToBytes(pk.curveP.BitLen()) } // Equals test the equality of two private keys -func (pk *pubKeyECDSA) Equals(other PublicKey) bool { - // check the key type - otherECDSA, ok := other.(*pubKeyECDSA) - if !ok { +func pubKeyCommonECDSAEquals(pk, other PublicKey) bool { + // check the algorithm + if pk.Algorithm() != other.Algorithm() { return false } - // check the curve - if pk.alg.curve != otherECDSA.alg.curve { - return false - } - return (pk.goPubKey.X.Cmp(otherECDSA.goPubKey.X) == 0) && - (pk.goPubKey.Y.Cmp(otherECDSA.goPubKey.Y) == 0) + // check the point + return bytes.Equal(pk.Encode(), other.Encode()) } -// String returns the hex string representation of the key. -func (pk *pubKeyECDSA) String() string { - return fmt.Sprintf("%#x", pk.Encode()) +// Helper function to pad two big integers to "size" bytes and concatenate them. +// This helper is needed in serializations in ECDSA implementation. +// It assumes the output buffer has at least 2*size byte-length +func padToSizeAndConcat(output []byte, a, b *big.Int, size int) { + a.FillBytes(output[:size]) + b.FillBytes(output[size:]) +} + +// Helper function to read two big integers of "size" bytes each from a concatenate input buffer. +// This helper is needed when deserializing. +// It assumes the input buffer has at least 2*size byte-length. +func readTwoBigInts(input []byte, size int) (*big.Int, *big.Int) { + a := new(big.Int).SetBytes(input[:size]) + b := new(big.Int).SetBytes(input[size : 2*size]) + return a, b } diff --git a/ecdsa_p256.go b/ecdsa_p256.go new file mode 100644 index 00000000..d50d76dc --- /dev/null +++ b/ecdsa_p256.go @@ -0,0 +1,275 @@ +/* + * Flow Crypto + * + * Copyright Flow Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package crypto + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "fmt" + + "github.com/onflow/crypto/hash" +) + +// ECDSA implementation on NIST-P256 is based on https://pkg.go.dev/crypto and https://pkg.go.dev/crypto/elliptic +// This implementation is not resistant against side-channel attacks or fault attacks. + +const ( + nLenP256 = 32 + pLenP256 = 32 +) + +var ( + // NIST P256 + SignatureLenECDSAP256 = 2 * nLenP256 + PrKeyLenECDSAP256 = nLenP256 + // PubKeyLenECDSAP256 is the size of uncompressed points on P256 + PubKeyLenECDSAP256 = 2 * pLenP256 +) + +// context of ECDSA on NIST P-256 +var p256Instance *ecdsaContext + +func initECDSAP256() { + curve := elliptic.P256() + p256Instance = &(ecdsaContext{ + curveP: curve.Params().P, + curveN: curve.Params().N, + algo: ECDSAP256, + }) +} + +// prKeyECDSAP256 is the private key of ECDSA on P256, it implements the interface PrivateKey +type prKeyECDSAP256 struct { + // ECDSA generic private key + *prKeyCommonECDSA + // go ecdsa standard lib private key + goPrKey *ecdsa.PrivateKey + // public key + pubKey *pubKeyECDSAP256 +} + +var _ PrivateKey = (*prKeyECDSAP256)(nil) + +// pubKeyECDSAP256 is the public key of ECDSA on P256, it implements PublicKey +type pubKeyECDSAP256 struct { + // ECDSA generic public key + *pubKeyCommonECDSA + // go ecdsa standard lib public key + goPubKey *ecdsa.PublicKey +} + +var _ PublicKey = (*pubKeyECDSAP256)(nil) + +func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error) { + internalSK, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), dBytes) + if err != nil { + return nil, fmt.Errorf("failed to parse raw private key: %w", err) + } + sk := &prKeyECDSAP256{ + prKeyCommonECDSA: &prKeyCommonECDSA{a}, + goPrKey: internalSK, + pubKey: nil, // public key is not constructed + } + return sk, nil +} + +// Sign signs an array of bytes +// +// The resulting signature is the concatenation bytes(r)||bytes(s), +// where r and s are padded to the curve order size. +// The private key is read only while sha2 and sha3 hashers are +// modified temporarily. +// +// The function returns: +// - (false, errNilHasher) if a hasher is nil +// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (nil, error) if an unexpected error occurs +// - (signature, nil) otherwise +func (sk *prKeyECDSAP256) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { + hash, err := sk.checkAlgoAndComputeHash(msg, hasher) + if err != nil { + return nil, err + } + r, s, err := ecdsa.Sign(rand.Reader, sk.goPrKey, hash) + if err != nil { + return nil, fmt.Errorf("ECDSA sign failed: %w", err) + } + + signature := make([]byte, 2*nLenP256) + padToSizeAndConcat(signature, r, s, nLenP256) + return signature, nil +} + +// String returns the hex string representation of the private key +func (sk *prKeyECDSAP256) String() string { + return prKeyCommonECDSAString(sk) +} + +// returns a publicKeyECDSAP256 from (bytes(x) || bytes(y)) bytes +func publicKeyECDSAP256(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSAP256, error) { + // deserialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3) + // and includes on curve check. + // The bytes serialization for non-infinity points is `0x04 || X || Y` and infinity point should be rejected anyway + parsingBytes := append([]byte{ecEncodingUncompressed}, XYBytes...) + + internalPK, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), parsingBytes) + if err != nil { + return nil, invalidInputsErrorf("input point has invalid coordinates or is not on curve: %w", err) + } + return &pubKeyECDSAP256{ + &pubKeyCommonECDSA{p256Instance}, + internalPK, + }, nil +} + +// String returns the hex string representation of the public key +func (pk *pubKeyECDSAP256) String() string { + return pubKeyCommonECDSAString(pk) +} + +// PublicKey returns the public key associated to the private key +func (sk *prKeyECDSAP256) PublicKey() PublicKey { + // construct the public key once + if sk.pubKey == nil { + sk.pubKey = &pubKeyECDSAP256{ + pubKeyCommonECDSA: &pubKeyCommonECDSA{p256Instance}, + goPubKey: &sk.goPrKey.PublicKey, + } + } + return sk.pubKey +} + +// Verify verifies a signature of an input data under the public key. +// +// If the input signature slice has an invalid length or fails to deserialize into valid +// scalars, the function returns false without an error. +// +// Public keys are read only, sha2 and sha3 hashers are +// modified temporarily. +// +// The function returns: +// - (false, errNilHasher) if a hasher is nil +// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (false, error) if an unexpected error occurs +// - (validity, nil) otherwise +func (pk *pubKeyECDSAP256) Verify(sig Signature, data []byte, alg hash.Hasher) (bool, error) { + h, err := pk.checkAlgoAndComputeHash(data, alg) + if err != nil { + return false, err + } + if len(sig) != SignatureLenECDSAP256 { + return false, nil + } + + r, s := readTwoBigInts(sig, nLenP256) + return ecdsa.Verify(pk.goPubKey, h, r, s), nil +} + +// given a private key (d), returns a raw encoding bytes(d) in big endian +// padded to the private key length +func (sk *prKeyECDSAP256) rawEncode() []byte { + skBytes, err := sk.goPrKey.Bytes() + if err != nil { + // not expected to happen since the private key is generated by this package and should be valid + panic(fmt.Sprintf("failed to encode private key: %v", err)) + } + return skBytes +} + +// Encode returns a byte representation of a private key. +// a simple raw byte encoding in big endian is used for all curves +func (sk *prKeyECDSAP256) Encode() []byte { + return sk.rawEncode() +} + +// Equals test the equality of two private keys +func (sk *prKeyECDSAP256) Equals(other PrivateKey) bool { + return prKeyCommonECDSAEquals(sk, other) +} + +// Equals test the equality of two public keys +func (pk *pubKeyECDSAP256) Equals(other PublicKey) bool { + return pubKeyCommonECDSAEquals(pk, other) +} + +// `rawEncode` returns a raw uncompressed encoding `bytes(x) || bytes(y)` given a public key (x,y). +// x and y are padded to the field size. +func (pk *pubKeyECDSAP256) rawEncode() []byte { + bytes, err := pk.goPubKey.Bytes() + if err != nil { + // not expected to happen since the public keys generated by this package only + // use elliptic.P256 + panic(fmt.Sprintf("unexpected failure to encode public key: %v", err)) + } + return bytes[1:] // remove the uncompressed point prefix +} + +// Encode returns a byte representation of a public key. +// a simple uncompressed raw encoding X||Y is used for all curves +// X and Y are the big endian byte encoding of the x and y coordinates of the public key +func (pk *pubKeyECDSAP256) Encode() []byte { + return pk.rawEncode() +} + +// EncodeCompressed returns a compressed encoding according to X9.62 section 4.3.6. +// This compressed representation uses an extra byte to disambiguate parity. +// The expected input is a public key (x,y). +// +// Receiver point is guaranteed to be on curve and to be non-infinity because +// the package does not allow constructing infinity points or points not on curve. +func (pk *pubKeyECDSAP256) EncodeCompressed() []byte { + bytes := pk.rawEncode() + // read X and Y from the encoding + x, y := readTwoBigInts(bytes, pLenP256) + // use elliptic.MarshalCompressed to get the compressed encoding + return elliptic.MarshalCompressed(elliptic.P256(), x, y) +} + +// p256DecodePublicKeyCompressed returns a non-infinity P-256 public key given the bytes of a compressed +// public key according to X9.62 section 4.3.6. +// Note that infinity point serialization isn't defined in this package so the input (or output) +// can never represent an infinity point. +// Error Returns: +// - invalidInputsError if the input isn't a valid key serialization +// on the given curve. +func p256DecodePublicKeyCompressed(pkBytes []byte) (*pubKeyECDSAP256, error) { + + expectedLen := pLenP256 + 1 + if len(pkBytes) != expectedLen { + return nil, invalidInputsErrorf("incorrect input length, expected %d, got %d", expectedLen, len(pkBytes)) + } + x, y := elliptic.UnmarshalCompressed(elliptic.P256(), pkBytes) + if x == nil || y == nil { + return nil, invalidInputsErrorf("input %x isn't a compressed serialization of a point on P256", pkBytes) + } + uncompressedPointBytes := make([]byte, 2*pLenP256+1) + uncompressedPointBytes[0] = ecEncodingUncompressed + padToSizeAndConcat(uncompressedPointBytes[1:], x, y, pLenP256) + + internalPK, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), uncompressedPointBytes) + if err != nil { + // unexpected error since prior deserialization succeeded + return nil, invalidInputsErrorf("unexpected error: input is not a point on curve P-256: %w", err) + } + return &pubKeyECDSAP256{ + &pubKeyCommonECDSA{p256Instance}, + internalPK, + }, nil +} diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go new file mode 100644 index 00000000..f23fdded --- /dev/null +++ b/ecdsa_secp256k1.go @@ -0,0 +1,267 @@ +/* + * Flow Crypto + * + * Copyright Flow Foundation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package crypto + +import ( + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/crypto/secp256k1" + + "github.com/onflow/crypto/hash" +) + +// ECDSA implementation on SECG secp256k1 is based on https://pkg.go.dev/github.com/ethereum/go-ethereum/crypto/secp256k1 + +// This implementation is not resistant against side-channel attacks or fault attacks. + +// curve parameters for SECG secp256k1 https://www.secg.org/sec2-v2.pdf +const ( + secp256k1PHex = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F" + secp256k1NHex = "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141" + + nLenSecp256k1 = 32 + pLenSecp256k1 = 32 +) + +var ( + // SECG secp256k1 + SignatureLenECDSASecp256k1 = 2 * nLenSecp256k1 + PrKeyLenECDSASecp256k1 = nLenSecp256k1 + // PubKeyLenECDSASecp256k1 is the size of uncompressed points on secp256k1 + PubKeyLenECDSASecp256k1 = 2 * pLenSecp256k1 +) + +// context of ECDSA on SECG secp256k1 curve https://www.secg.org/sec2-v2.pdf +var secp256k1Instance *ecdsaContext + +func initECDSASecp256k1() { + curveP, ok := new(big.Int).SetString(secp256k1PHex, 16) + if !ok { + panic("failed to initialize ECDSA with secp256k1 curve") + } + curveN, ok := new(big.Int).SetString(secp256k1NHex, 16) + if !ok { + panic("failed to initialize ECDSA with secp256k1 curve") + } + secp256k1Instance = &(ecdsaContext{ + curveP: curveP, + curveN: curveN, + algo: ECDSASecp256k1, + }) +} + +// prKeyECDSASecp256k1 is the private key of ECDSA on SECG secp256k1, it implements PrivateKey +type prKeyECDSASecp256k1 struct { + // ECDSA generic private key + *prKeyCommonECDSA + // bytes(D) of private scalar D in big endian, padded to the curve order size (32 bytes) + dBytes []byte + // public key + pubKey *pubKeyECDSASecp256k1 +} + +var _ PrivateKey = (*prKeyECDSASecp256k1)(nil) + +// pubKeyECDSASecp256k1 is the public key of ECDSA on SECG secp256k1, it implements PublicKey +type pubKeyECDSASecp256k1 struct { + // ECDSA generic public key + *pubKeyCommonECDSA + // 0x4 || bytes(x) || bytes(y) (65 bytes) where x and y are the coordinates of the public key point, padded to the field size (32 bytes) + pkBytes []byte +} + +var _ PublicKey = (*pubKeyECDSASecp256k1)(nil) + +func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256k1 { + sk := &prKeyECDSASecp256k1{ + prKeyCommonECDSA: &prKeyCommonECDSA{a}, + dBytes: dBytes, + pubKey: nil, // public key is not constructed + } + return sk +} + +// Sign signs an array of bytes +// +// The resulting signature is the concatenation bytes(r)||bytes(s), +// where r and s are padded to the curve order size. +// The private key is read only while sha2 and sha3 hashers are +// modified temporarily. +// +// The function returns: +// - (false, errNilHasher) if a hasher is nil +// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (nil, error) if an unexpected error occurs +// - (signature, nil) otherwise +func (sk *prKeyECDSASecp256k1) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { + hash, err := sk.checkAlgoAndComputeHash(msg, hasher) + if err != nil { + return nil, err + } + // truncate the hash to the curve order size, as specified in FIPS 186-4 section 6.4 + // and as required by the secp256k1 package signing function + hash = hash[:nLenSecp256k1] + signature, err := secp256k1.Sign(hash, sk.dBytes) + if err != nil { + return nil, fmt.Errorf("failed to sign hash: %w", err) + } + // remove the EC recover byte (last byte) + return signature[:SignatureLenECDSASecp256k1], nil +} + +// String returns the hex string representation of the private key +func (sk *prKeyECDSASecp256k1) String() string { + return prKeyCommonECDSAString(sk) +} + +func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp256k1, error) { + pLen := bitsToBytes(a.curveP.BitLen()) + x, y := readTwoBigInts(XYBytes, pLen) + // `IsOnCurve` is not deprecated in btcec's type `KoblitzCurve` + if !secp256k1.S256().IsOnCurve(x, y) { + return nil, invalidInputsErrorf("input point has invalid coordinates or is not on curve") + } + return &pubKeyECDSASecp256k1{ + &pubKeyCommonECDSA{secp256k1Instance}, + append([]byte{ecEncodingUncompressed}, XYBytes...), + // XYBytes is already the raw uncompressed encoding `bytes(x) || bytes(y)` + }, nil +} + +// String returns the hex string representation of the public key +func (pk *pubKeyECDSASecp256k1) String() string { + return pubKeyCommonECDSAString(pk) +} + +// PublicKey returns the public key associated to the private key +func (sk *prKeyECDSASecp256k1) PublicKey() PublicKey { + // construct the public key once + if sk.pubKey == nil { + x, y := secp256k1.S256().ScalarBaseMult(sk.dBytes) + pkBytes := make([]byte, 1+2*pLenSecp256k1) + pkBytes[0] = ecEncodingUncompressed + // pad x and y to the field size and concatenate them + padToSizeAndConcat(pkBytes[1:], x, y, pLenSecp256k1) + sk.pubKey = &pubKeyECDSASecp256k1{ + pubKeyCommonECDSA: &pubKeyCommonECDSA{secp256k1Instance}, + pkBytes: pkBytes, + } + } + return sk.pubKey +} + +// Verify verifies a signature of an input data under the public key. +// +// If the input signature slice has an invalid length or fails to deserialize into valid +// scalars, the function returns false without an error. +// +// Public keys are read only, sha2 and sha3 hashers are +// modified temporarily. +// +// The function returns: +// - (false, errNilHasher) if a hasher is nil +// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (false, error) if an unexpected error occurs +// - (validity, nil) otherwise +func (pk *pubKeyECDSASecp256k1) Verify(sig Signature, msg []byte, hasher hash.Hasher) (bool, error) { + hash, err := pk.checkAlgoAndComputeHash(msg, hasher) + if err != nil { + return false, err + } + if len(sig) != 2*nLenSecp256k1 { + return false, nil + } + + return secp256k1.VerifySignature(pk.pkBytes, hash, sig), nil +} + +// given a private key (d), returns a raw encoding bytes(d) in big endian +// padded to the private key length +func (sk *prKeyECDSASecp256k1) rawEncode() []byte { + return sk.dBytes +} + +// Encode returns a byte representation of a private key. +// a simple raw byte encoding in big endian is used for all curves +func (sk *prKeyECDSASecp256k1) Encode() []byte { + return sk.rawEncode() +} + +// Equals test the equality of two private keys +func (sk *prKeyECDSASecp256k1) Equals(other PrivateKey) bool { + return prKeyCommonECDSAEquals(sk, other) +} + +// Equals test the equality of two public keys +func (pk *pubKeyECDSASecp256k1) Equals(other PublicKey) bool { + return pubKeyCommonECDSAEquals(pk, other) +} + +// `rawEncode` returns a raw uncompressed encoding `bytes(x) || bytes(y)` given a public key (x,y). +// x and y are padded to the field size. +func (pk *pubKeyECDSASecp256k1) rawEncode() []byte { + // skip the uncompressed encoding byte + return pk.pkBytes[1:] +} + +// Encode returns a byte representation of a public key. +// a simple uncompressed raw encoding X||Y is used for all curves +// X and Y are the big endian byte encoding of the x and y coordinates of the public key +func (pk *pubKeyECDSASecp256k1) Encode() []byte { + return pk.rawEncode() +} + +// EncodeCompressed returns a compressed encoding according to X9.62 section 4.3.6. +// This compressed representation uses an extra byte to disambiguate parity. +// The expected input is a public key (x,y). +// +// Receiver point is guaranteed to be on curve and to be non-infinity because +// the package does not allow constructing infinity points or points not on curve. +func (pk *pubKeyECDSASecp256k1) EncodeCompressed() []byte { + x, y := readTwoBigInts(pk.pkBytes[1:], pLenSecp256k1) + // read X and Y from the encoding + return secp256k1.CompressPubkey(x, y) +} + +// p256DecodePublicKeyCompressed returns a non-infinity P-256 public key given the bytes of a compressed +// public key according to X9.62 section 4.3.6. +// Note that infinity point serialization isn't defined in this package so the input (or output) +// can never represent an infinity point. +// Error Returns: +// - invalidInputsError if the input isn't a valid key serialization +// on the given curve. +func secp256k1DecodePublicKeyCompressed(pkBytes []byte) (*pubKeyECDSASecp256k1, error) { + expectedLen := pLenSecp256k1 + 1 + if len(pkBytes) != expectedLen { + return nil, invalidInputsErrorf("incorrect input length, expected %d, got %d", expectedLen, len(pkBytes)) + } + x, y := secp256k1.DecompressPubkey(pkBytes) + if x == nil || y == nil { + return nil, invalidInputsErrorf("input %x isn't a compressed serialization of a point on secp256k1", pkBytes) + } + uncompressedPkBytes := make([]byte, 1+2*pLenSecp256k1) + uncompressedPkBytes[0] = ecEncodingUncompressed + padToSizeAndConcat(uncompressedPkBytes[1:], x, y, pLenSecp256k1) + + return &pubKeyECDSASecp256k1{ + &pubKeyCommonECDSA{secp256k1Instance}, + uncompressedPkBytes, + }, nil +} diff --git a/ecdsa_test.go b/ecdsa_test.go index 30ba13d4..970e5203 100644 --- a/ecdsa_test.go +++ b/ecdsa_test.go @@ -24,7 +24,6 @@ import ( "crypto/elliptic" crand "crypto/rand" - "math/big" "github.com/btcsuite/btcd/btcec/v2" "github.com/stretchr/testify/assert" @@ -178,7 +177,6 @@ func TestECDSAEncodeDecode(t *testing.T) { pk, err := DecodePublicKey(curve, pkBytes) require.Error(t, err, "point is not on curve") assert.True(t, IsInvalidInputsError(err)) - assert.ErrorContains(t, err, "input is not a point on curve") assert.Nil(t, pk) }) @@ -201,13 +199,11 @@ func TestECDSAEncodeDecode(t *testing.T) { require.NoError(t, err) _, err = DecodePublicKey(curve, invalidPk1) assert.Error(t, err) - assert.ErrorContains(t, err, "at least one coordinate is larger than the field prime for") // invalidpk2 with y >= p invalidPk2, err := hex.DecodeString(invalidPK2s[curve]) require.NoError(t, err) _, err = DecodePublicKey(curve, invalidPk2) assert.Error(t, err) - assert.ErrorContains(t, err, "at least one coordinate is larger than the field prime for") }) } } @@ -270,52 +266,7 @@ func TestECDSAPublicKeyComputation(t *testing.T) { } } -// TestGoECDSAP256PrivateKeyConstruction exercises `goecdsaPrivateKey` for P-256 directly. -// -// This is a regression test for a panic that surfaced on Go 1.26: -// when constructing the key, the public affine coordinates `X`/`Y` are still nil -// (we are in the middle of computing them via base scalar multiplication). -// Since Go 1.26, `(*ecdsa.PrivateKey).ECDH` serializes the key through -// `(*PrivateKey).Bytes`, which reads `X`/`Y` and therefore panicked on the nil deref. -// The fix builds the ecdh key from the scalar bytes instead. -// -// The test asserts the construction does not panic and that the computed public key -// matches a known test vector, so it guards both the crash and the correctness of the -// scalar-based code path. -func TestGoECDSAP256PrivateKeyConstruction(t *testing.T) { - // scalar / expected public key pair, identical to the P-256 vector in - // TestECDSAPublicKeyComputation - const skHex = "6e37a39c31a05181bf77919ace790efd0bdbcaf42b5a52871fc112fceb918c95" - const xHex = "78a80dfe190a6068be8ddf05644c32d2540402ffc682442f6a9eeb96125d8681" - const yHex = "3789f92cf4afabf719aaba79ecec54b27e33a188f83158f6dd15ecb231b49808" - - skBytes, err := hex.DecodeString(skHex) - require.NoError(t, err) - d := new(big.Int).SetBytes(skBytes) - - // the call panicked on Go 1.26 before the fix - require.NotPanics(t, func() { - priv, err := goecdsaPrivateKey(elliptic.P256(), d) - require.NoError(t, err) - require.NotNil(t, priv) - - // the scalar must be preserved - assert.Equal(t, 0, priv.D.Cmp(d)) - - // the computed public affine coordinates must match the known vector - expectedX, ok := new(big.Int).SetString(xHex, 16) - require.True(t, ok) - expectedY, ok := new(big.Int).SetString(yHex, 16) - require.True(t, ok) - assert.Equal(t, 0, priv.PublicKey.X.Cmp(expectedX)) - assert.Equal(t, 0, priv.PublicKey.Y.Cmp(expectedY)) - - // the computed point must be on the curve - assert.True(t, elliptic.P256().IsOnCurve(priv.PublicKey.X, priv.PublicKey.Y)) - }) -} - -func TestSignatureFormatCheck(t *testing.T) { +func TestECDSASignatureFormatCheck(t *testing.T) { for _, curve := range ecdsaCurves { t.Run("valid signature", func(t *testing.T) { diff --git a/go.mod b/go.mod index 344580d9..4e27765d 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.26.0 require ( github.com/btcsuite/btcd/btcec/v2 v2.3.4 + github.com/ethereum/go-ethereum v1.17.5 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.36.0 + github.com/stretchr/testify v1.11.1 + golang.org/x/crypto v0.54.0 gonum.org/v1/gonum v0.16.0 pgregory.net/rapid v0.4.7 ) @@ -14,9 +15,8 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.31.0 // indirect + golang.org/x/sys v0.47.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 16ac77ab..3e14abed 100644 --- a/go.sum +++ b/go.sum @@ -6,24 +6,27 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/ethereum/go-ethereum v1.17.5 h1:o9BIXs2Q/3cPHVxw49n+Zjn2i6rB9TOXatev46duOC4= +github.com/ethereum/go-ethereum v1.17.5/go.mod h1:vz2YvG7RewA4sFHTgzLyW+WmFG1N4jfk/hgXQVhhn9c= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sign.go b/sign.go index 9d5000fe..3d368879 100644 --- a/sign.go +++ b/sign.go @@ -19,11 +19,8 @@ package crypto import ( - "crypto/elliptic" "fmt" - "github.com/btcsuite/btcd/btcec/v2" - "github.com/onflow/crypto/hash" ) @@ -83,20 +80,10 @@ func newSigner(algo SigningAlgorithm) (signer, error) { // Initialize the context of all algos func init() { // ECDSA - p256Instance = &(ecdsaAlgo{ - curve: elliptic.P256(), - algo: ECDSAP256, - }) - secp256k1Instance = &(ecdsaAlgo{ - curve: btcec.S256(), - algo: ECDSASecp256k1, - }) + initECDSA() // BLS initBLS12381() - blsInstance = &blsBLS12381Algo{ - algo: BLSBLS12381, - } } // SignatureFormatCheck verifies the format of a serialized signature, From 16c5655f1b2aa2cf43689321203ee4914531e935 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 13:59:20 -0500 Subject: [PATCH 03/21] towards isolaing ECDSA secp256k1 in cgo mode only --- ecdsa_secp256k1.go | 2 ++ no_cgo.go | 29 +++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index f23fdded..a2c75fd6 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -1,3 +1,5 @@ +//go:build cgo && !no_cgo + /* * Flow Crypto * diff --git a/no_cgo.go b/no_cgo.go index fd95f65c..aae3d51a 100644 --- a/no_cgo.go +++ b/no_cgo.go @@ -15,9 +15,12 @@ import ( ) const ( - SignatureLenBLSBLS12381 = 0 - PubKeyLenBLSBLS12381 = 0 - PrKeyLenBLSBLS12381 = 0 + SignatureLenBLSBLS12381 = 0 + PubKeyLenBLSBLS12381 = 0 + PrKeyLenBLSBLS12381 = 0 + SignatureLenECDSASecp256k1 = 0 + PrKeyLenECDSASecp256k1 = 0 + PubKeyLenECDSASecp256k1 = 0 ) func initBLS12381() {} @@ -30,7 +33,6 @@ type blsBLS12381Algo struct { algo SigningAlgorithm } -// BLS context on the BLS 12-381 curve var blsInstance *blsBLS12381Algo func (a *blsBLS12381Algo) generatePrivateKey(ikm []byte) (PrivateKey, error) { @@ -192,3 +194,22 @@ func IsNotBLSKeyError(err error) bool { func IsInvalidSignatureError(err error) bool { panic(withFeature("BLS multi-sig")) } + +func initECDSASecp256k1() { + panic(withFeature("ECDSA SECP256k1")) +} + +type pubKeyECDSASecp256k1 struct{} +type prKeyECDSASecp256k1 struct{} + +func secp256k1DecodePublicKeyCompressed(pkBytes []byte) (*pubKeyECDSASecp256k1, error) { + panic(withFeature("ECDSA SECP256k1")) +} + +func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256k1 { + panic(withFeature("ECDSA SECP256k1")) +} + +func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp256k1, error) { + panic(withFeature("ECDSA SECP256k1")) +} From 975a8e0b26435812a8b4b0367c1aeb24eb802fe1 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 14:00:40 -0500 Subject: [PATCH 04/21] revert linter exception --- .golangci.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index d73fb864..ca737b81 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -10,17 +10,6 @@ linters: - linters: - govet text: "unsafeptr" # disable flagging unsafeptr usage - # ecdsa.go wraps crypto/ecdsa, whose Sign/Verify still consume the raw - # PrivateKey.D and PublicKey.X/Y fields. - # Go 1.26 deprecated direct access to those fields, but the recommended - # replacement API (ecdsa.ParseRawPrivateKey / ParseUncompressedPublicKey / - # (*PrivateKey).Bytes / (*PublicKey).Bytes) supports only the NIST curves and - # rejects secp256k1, which this package supports via btcec's custom - # elliptic.Curve, so the package has to keep using the low-level fields. - - path: (^|/)ecdsa(_test)?\.go$ - linters: - - staticcheck - text: "SA1019" formatters: exclusions: paths: From 430b1ea81d53f7aa7e3f38bedd4d6c444ab1ab66 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 14:38:07 -0500 Subject: [PATCH 05/21] remove non-cgo mode --- .github/workflows/ci.yml | 11 +- Makefile | 8 -- README.md | 28 +---- bls.go | 2 - bls12381_utils.go | 2 - bls12381_utils_test.go | 2 - bls_crossBLST_test.go | 2 - bls_multisig.go | 2 - bls_test.go | 2 - bls_thresholdsign.go | 2 - bls_thresholdsign_test.go | 2 - dkg_feldmanvss.go | 2 - dkg_feldmanvssq.go | 2 - dkg_jointfeldman.go | 2 - dkg_test.go | 2 - ecdsa_secp256k1.go | 2 - no_cgo.go | 215 -------------------------------------- no_cgo_test.go | 46 -------- spock.go | 2 - spock_test.go | 2 - 20 files changed, 6 insertions(+), 332 deletions(-) delete mode 100644 no_cgo.go delete mode 100644 no_cgo_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a4f66357..53141423 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: cache: true - name: run Go tidy run: make go-tidy - - name: Run golangci-lint with cgo + - name: Run golangci-lint env: CGO_ENABLED: 1 uses: golangci/golangci-lint-action@v8 @@ -42,15 +42,6 @@ jobs: version: ${{ env.LINT_VERSION }} # https://github.com/golangci/golangci-lint-action/issues/244 skip-cache: true - - name: Run golangci-lint without cgo - env: - CGO_ENABLED: 0 - uses: golangci/golangci-lint-action@v8 - with: - version: ${{ env.LINT_VERSION }} - args: --build-tags no_cgo - # https://github.com/golangci/golangci-lint-action/issues/244 - skip-cache: true - name: Run Go Fix run: make go-fix - name: Run incorrect builds diff --git a/Makefile b/Makefile index 1fdd5cd5..aba22695 100644 --- a/Makefile +++ b/Makefile @@ -96,15 +96,7 @@ go-lint: go-tidy go-fix test: # root package CGO_ENABLED=1 CGO_CFLAGS=$(ADX_FLAG) go test -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(VERBOSE),-v,) -#root package without cgo - CGO_ENABLED=0 go test -tags=no_cgo -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(VERBOSE),-v,) # sub packages go test -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(VERBOSE),-v,) ./hash go test -coverprofile=$(COVER_PROFILE) $(RACE_FLAG) $(if $(JSON_OUTPUT),-json,) $(if $(VERBOSE),-v,) ./random -# test incorrect builds and make sure they fail -.PHONY: incorrect_builds -incorrect_builds: -# both tests should fail - ! CGO_ENABLED=0 go test - ! CGO_ENABLED=1 CGO_CFLAGS=$(ADX_FLAG) go test -tags=no_cgo diff --git a/README.md b/README.md index 0c78ecac..4b912d94 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,6 @@ import "github.com/onflow/crypto" Building your project with Flow crypto and enabling all the supported algorithms requires using cgo to compile the C code underneath. If cgo isn't enabled by default, the `CGO_ENABLED` environment variable should be set to `1`. -It is also possible to build without cgo (`CGO_ENABLED=0`) but this would disable some primitives (the ones related to BLS). - -### Build with cgo - -Building with cgo is required to support all the algorithms of the module, including the algorithms based on the BLS12-381 curve. If the test or target application crashes with a "Caught SIGILL" exception, rebuild with `CGO_CFLAGS` set to `"-O2 -D__BLST_PORTABLE__"` to disable non-portable code. The runtime error can happen if the CPU doesn't support certain instructions. @@ -53,19 +48,6 @@ GOOS=linux GOARCH=arm64 CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 go build When using the `go mod vendor` command in your project, [a known issue](https://github.com/golang/go/issues/26366) with the Go vendoring tool prevents cgo dependencies from being copied into your vendor directory. This results in build errors related to the Flow crypto package. External vendoring tools that do copy the entire package files can be used instead of the Go command to resolve the issue. - -### Build without cgo - -It is possible to build without cgo but this requires disabling all primitives based on the BLS12-381 curve (BLS signature, BLS threshold signature, BLS-based DKG, BLS-based SPoCK). -Refer to [algorithms](#algorithms) and [protocols](#protocols) to check the supported features. -Calling any of the non-supported primitives would panic. -In order to avoid accidental builds that result in unwanted crashes, disabling cgo must be confirmed with the `no_cgo` build tag. - -``` -CGO_ENABLED=0 go build -tags=no_cgo -``` - - ## Algorithms ### Hashing and MAC: @@ -86,7 +68,7 @@ All signature schemes use the generic interfaces of `PrivateKey` and `PublicKey` * ephemeral key is derived from the private key, hash and the system entropy (based on https://golang.org/pkg/crypto/ecdsa/). * supports NIST P-256 (secp256r1) and secp256k1 curves. - * BLS (requires cgo) + * BLS * supports [BLS12-381](https://electriccoin.co/blog/new-snark-curve/) curve. * is implementing the minimal-signature-size variant: signatures in G1 and public keys in G2. @@ -114,7 +96,7 @@ All signature schemes use the generic interfaces of `PrivateKey` and `PublicKey` ### Threshold Signature - * BLS-based threshold signature (requires cgo) + * BLS-based threshold signature * [non interactive](https://www.iacr.org/archive/pkc2003/25670031/25670031.pdf) threshold signature reconstruction. * supports only BLS 12-381 curve with the same features above. * (t+1) signatures are required to reconstruct the threshold signature. @@ -126,16 +108,16 @@ All signature schemes use the generic interfaces of `PrivateKey` and `PublicKey` All supported Distributed Key Generation protocols are [discrete log based](http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.50.2737&rep=rep1&type=pdf) and are implemented for the same BLS setup on the BLS 12-381 curve. The protocols generate key sets for the BLS-based threshold signature. - * Feldman VSS (requires cgo) + * Feldman VSS * simple verifiable secret sharing with a single dealer. * the library does not implement the communication channels between participants. The caller should implement the methods `PrivateSend` (1-to-1 messaging) and `Broadcast` (1-to-n messaging) * 1-to-1 messaging must be a private channel, the caller must make sure the channel preserves confidentialiy and authenticates the sender. * 1-to-n broadcasting is a reliable broadcast, where honest senders are able to reach all honest receivers, and where all honest receivers end up with the same received messages. The channel should also authenticate the broadcaster. * It is recommended that both communication channels are unique per protocol instance. This could be achieved by prepending the messages to send/broadcast by a unique protocol instance ID. - * Feldman VSS Qual (requires cgo) + * Feldman VSS Qual * an extension of the simple Feldman VSS. * implements a complaint mechanism to qualify/disqualify the dealer. - * Joint Feldman (Pedersen) (requires cgo) + * Joint Feldman (Pedersen) * distributed generation. * based on parallel instances of Feldman VSS Qual, each with a different dealer. * same assumptions about the communication channels as in Feldman VSS. diff --git a/bls.go b/bls.go index e371c60e..e80465cf 100644 --- a/bls.go +++ b/bls.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls12381_utils.go b/bls12381_utils.go index aa939c79..784f5389 100644 --- a/bls12381_utils.go +++ b/bls12381_utils.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls12381_utils_test.go b/bls12381_utils_test.go index f392be05..35a37025 100644 --- a/bls12381_utils_test.go +++ b/bls12381_utils_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls_crossBLST_test.go b/bls_crossBLST_test.go index b5f47f60..a0d04b60 100644 --- a/bls_crossBLST_test.go +++ b/bls_crossBLST_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls_multisig.go b/bls_multisig.go index dbb1625e..03f55e6e 100644 --- a/bls_multisig.go +++ b/bls_multisig.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls_test.go b/bls_test.go index f7e7893d..4fbf4fe4 100644 --- a/bls_test.go +++ b/bls_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls_thresholdsign.go b/bls_thresholdsign.go index c7de6761..f790df7f 100644 --- a/bls_thresholdsign.go +++ b/bls_thresholdsign.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/bls_thresholdsign_test.go b/bls_thresholdsign_test.go index a69155bf..f8c310f8 100644 --- a/bls_thresholdsign_test.go +++ b/bls_thresholdsign_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/dkg_feldmanvss.go b/dkg_feldmanvss.go index 696e9050..959da868 100644 --- a/dkg_feldmanvss.go +++ b/dkg_feldmanvss.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/dkg_feldmanvssq.go b/dkg_feldmanvssq.go index 61e81049..bc1bf72a 100644 --- a/dkg_feldmanvssq.go +++ b/dkg_feldmanvssq.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/dkg_jointfeldman.go b/dkg_jointfeldman.go index 3f82f5b6..14e05c6c 100644 --- a/dkg_jointfeldman.go +++ b/dkg_jointfeldman.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/dkg_test.go b/dkg_test.go index e225d5b9..7d28c546 100644 --- a/dkg_test.go +++ b/dkg_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index a2c75fd6..f23fdded 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/no_cgo.go b/no_cgo.go deleted file mode 100644 index aae3d51a..00000000 --- a/no_cgo.go +++ /dev/null @@ -1,215 +0,0 @@ -//go:build !cgo && no_cgo - -package crypto - -// This file enables the build of the library when cgo is disabled, i.e when the environment -// variable `CGO_ENABLED` is set to `0`. -// The build without cgo succeeds but disables all algorithms working with -// the BLS12-381 curve (BLS signature, BLS threshold signature, BLS-based DKG and BLS-SPoCK). -// Any call to any of these algorithms would panic. - -import ( - "fmt" - - "github.com/onflow/crypto/hash" -) - -const ( - SignatureLenBLSBLS12381 = 0 - PubKeyLenBLSBLS12381 = 0 - PrKeyLenBLSBLS12381 = 0 - SignatureLenECDSASecp256k1 = 0 - PrKeyLenECDSASecp256k1 = 0 - PubKeyLenECDSASecp256k1 = 0 -) - -func initBLS12381() {} - -func withFeature(feature string) string { - return fmt.Sprintf("%s is only supported with cgo, rebuild with CGO_ENABLED=1\n", feature) -} - -type blsBLS12381Algo struct { - algo SigningAlgorithm -} - -var blsInstance *blsBLS12381Algo - -func (a *blsBLS12381Algo) generatePrivateKey(ikm []byte) (PrivateKey, error) { - panic(withFeature("BLS signature")) -} - -func (a *blsBLS12381Algo) decodePrivateKey(privateKeyBytes []byte) (PrivateKey, error) { - panic(withFeature("BLS signature")) -} - -func (a *blsBLS12381Algo) decodePublicKey(publicKeyBytes []byte) (PublicKey, error) { - panic(withFeature("BLS signature")) -} - -func (a *blsBLS12381Algo) decodePublicKeyCompressed(publicKeyBytes []byte) (PublicKey, error) { - panic(withFeature("BLS signature")) -} - -func NewExpandMsgXOFKMAC128(domainTag string) hash.Hasher { - panic(withFeature("BLS hasher")) -} - -func IsBLSSignatureIdentity(s Signature) bool { - panic(withFeature("BLS signature")) -} - -func BLSInvalidSignature() Signature { - panic(withFeature("BLS signature")) -} - -func isG2Compressed() bool { - panic(withFeature("BLS12-381 curve")) -} - -func NewBLSThresholdSignatureParticipant( - groupPublicKey PublicKey, - sharePublicKeys []PublicKey, - threshold int, - myIndex int, - myPrivateKey PrivateKey, - message []byte, - dsTag string, -) (ThresholdSignatureParticipant, error) { - panic(withFeature("BLS threshold signature")) -} - -func NewBLSThresholdSignatureInspector( - groupPublicKey PublicKey, - sharePublicKeys []PublicKey, - threshold int, - message []byte, - dsTag string, -) (ThresholdSignatureInspector, error) { - panic(withFeature("BLS threshold signature")) -} - -func BLSReconstructThresholdSignature(size int, threshold int, - shares []Signature, signers []int) (Signature, error) { - _ = duplicatedSignerErrorf("") - _ = notEnoughSharesErrorf("") - panic(withFeature("BLS threshold signature")) -} - -func EnoughShares(threshold int, sharesNumber int) (bool, error) { - panic(withFeature("BLS threshold signature")) -} - -func BLSThresholdKeyGen(size int, threshold int, seed []byte) ([]PrivateKey, - []PublicKey, PublicKey, error) { - panic(withFeature("BLS threshold signature")) -} - -func NewFeldmanVSS(size int, threshold int, myIndex int, - processor DKGProcessor, dealerIndex int) (DKGState, error) { - _, _ = newDKGCommon(size, threshold, myIndex, - processor, dealerIndex) - panic(withFeature("BLS-DKG")) -} - -func NewFeldmanVSSQual(size int, threshold int, myIndex int, - processor DKGProcessor, dealerIndex int) (DKGState, error) { - _ = dkgFailureErrorf("") - _ = dkgInvalidStateTransitionErrorf("") - panic(withFeature("BLS-DKG")) -} - -func NewJointFeldman(size int, threshold int, myIndex int, - processor DKGProcessor) (DKGState, error) { - _ = feldmanVSSShare | feldmanVSSVerifVec | feldmanVSSComplaint | feldmanVSSComplaintAnswer - panic(withFeature("BLS-DKG")) -} - -func SPOCKProve(sk PrivateKey, data []byte, kmac hash.Hasher) (Signature, error) { - panic(withFeature("BLS-SPoCK")) -} - -func SPOCKVerifyAgainstData(pk PublicKey, proof Signature, data []byte, kmac hash.Hasher) (bool, error) { - panic(withFeature("BLS-SPoCK")) -} - -func SPOCKVerify(pk1 PublicKey, proof1 Signature, pk2 PublicKey, proof2 Signature) (bool, error) { - panic(withFeature("BLS-SPoCK")) -} - -func BLSGeneratePOP(sk PrivateKey) (Signature, error) { - panic(withFeature("BLS multi-sig")) -} - -func BLSVerifyPOP(pk PublicKey, s Signature) (bool, error) { - panic(withFeature("BLS multi-sig")) -} - -func AggregateBLSSignatures(sigs []Signature) (Signature, error) { - panic(withFeature("BLS multi-sig")) -} - -func AggregateBLSPrivateKeys(keys []PrivateKey) (PrivateKey, error) { - panic(withFeature("BLS multi-sig")) -} - -func AggregateBLSPublicKeys(keys []PublicKey) (PublicKey, error) { - panic(withFeature("BLS multi-sig")) -} - -func IdentityBLSPublicKey() PublicKey { - panic(withFeature("BLS multi-sig")) -} - -func RemoveBLSPublicKeys(aggKey PublicKey, keysToRemove []PublicKey) (PublicKey, error) { - panic(withFeature("BLS multi-sig")) -} - -func VerifyBLSSignatureOneMessage( - pks []PublicKey, s Signature, message []byte, kmac hash.Hasher, -) (bool, error) { - panic(withFeature("BLS multi-sig")) -} - -func VerifyBLSSignatureManyMessages( - pks []PublicKey, s Signature, messages [][]byte, kmac []hash.Hasher, -) (bool, error) { - panic(withFeature("BLS multi-sig")) -} - -func BatchVerifyBLSSignaturesOneMessage( - pks []PublicKey, sigs []Signature, message []byte, kmac hash.Hasher, -) ([]bool, error) { - panic(withFeature("BLS multi-sig")) -} - -func IsBLSAggregateEmptyListError(err error) bool { - panic(withFeature("BLS multi-sig")) -} - -func IsNotBLSKeyError(err error) bool { - panic(withFeature("BLS multi-sig")) -} - -func IsInvalidSignatureError(err error) bool { - panic(withFeature("BLS multi-sig")) -} - -func initECDSASecp256k1() { - panic(withFeature("ECDSA SECP256k1")) -} - -type pubKeyECDSASecp256k1 struct{} -type prKeyECDSASecp256k1 struct{} - -func secp256k1DecodePublicKeyCompressed(pkBytes []byte) (*pubKeyECDSASecp256k1, error) { - panic(withFeature("ECDSA SECP256k1")) -} - -func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256k1 { - panic(withFeature("ECDSA SECP256k1")) -} - -func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp256k1, error) { - panic(withFeature("ECDSA SECP256k1")) -} diff --git a/no_cgo_test.go b/no_cgo_test.go deleted file mode 100644 index efc32a29..00000000 --- a/no_cgo_test.go +++ /dev/null @@ -1,46 +0,0 @@ -//go:build !cgo && no_cgo - -package crypto_test - -import ( - "testing" - - "github.com/onflow/crypto" - "github.com/stretchr/testify/assert" -) - -// Test all public functions requiring cgo. -// These functions must panic if built without cgo. -func TestNoRelicPanic(t *testing.T) { - assert.Panics(t, func() { _, _ = crypto.GeneratePrivateKey(crypto.BLSBLS12381, nil) }) - assert.Panics(t, func() { _, _ = crypto.DecodePrivateKey(crypto.BLSBLS12381, nil) }) - assert.Panics(t, func() { _, _ = crypto.DecodePublicKey(crypto.BLSBLS12381, nil) }) - assert.Panics(t, func() { _, _ = crypto.DecodePublicKeyCompressed(crypto.BLSBLS12381, nil) }) - assert.Panics(t, func() { _ = crypto.NewExpandMsgXOFKMAC128("") }) - assert.Panics(t, func() { _ = crypto.BLSInvalidSignature() }) - assert.Panics(t, func() { _, _ = crypto.BLSGeneratePOP(nil) }) - assert.Panics(t, func() { _, _ = crypto.BLSVerifyPOP(nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.AggregateBLSSignatures(nil) }) - assert.Panics(t, func() { _, _ = crypto.AggregateBLSPrivateKeys(nil) }) - assert.Panics(t, func() { _, _ = crypto.AggregateBLSPublicKeys(nil) }) - assert.Panics(t, func() { _ = crypto.IdentityBLSPublicKey() }) - assert.Panics(t, func() { _ = crypto.IsBLSAggregateEmptyListError(nil) }) - assert.Panics(t, func() { _ = crypto.IsInvalidSignatureError(nil) }) - assert.Panics(t, func() { _ = crypto.IsNotBLSKeyError(nil) }) - assert.Panics(t, func() { _ = crypto.IsBLSSignatureIdentity(nil) }) - assert.Panics(t, func() { _, _ = crypto.RemoveBLSPublicKeys(nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.VerifyBLSSignatureOneMessage(nil, nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.VerifyBLSSignatureManyMessages(nil, nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.BatchVerifyBLSSignaturesOneMessage(nil, nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.SPOCKProve(nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.SPOCKVerify(nil, nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.SPOCKVerifyAgainstData(nil, nil, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.NewBLSThresholdSignatureParticipant(nil, nil, 0, 0, nil, nil, "") }) - assert.Panics(t, func() { _, _ = crypto.NewBLSThresholdSignatureInspector(nil, nil, 0, nil, "") }) - assert.Panics(t, func() { _, _ = crypto.BLSReconstructThresholdSignature(0, 0, nil, nil) }) - assert.Panics(t, func() { _, _ = crypto.EnoughShares(0, 0) }) - assert.Panics(t, func() { _, _, _, _ = crypto.BLSThresholdKeyGen(0, 0, nil) }) - assert.Panics(t, func() { _, _ = crypto.NewFeldmanVSS(0, 0, 0, nil, 0) }) - assert.Panics(t, func() { _, _ = crypto.NewFeldmanVSSQual(0, 0, 0, nil, 0) }) - assert.Panics(t, func() { _, _ = crypto.NewJointFeldman(0, 0, 0, nil) }) -} diff --git a/spock.go b/spock.go index 96f1a593..5927c9be 100644 --- a/spock.go +++ b/spock.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * diff --git a/spock_test.go b/spock_test.go index 1bf8d808..9efccf30 100644 --- a/spock_test.go +++ b/spock_test.go @@ -1,5 +1,3 @@ -//go:build cgo && !no_cgo - /* * Flow Crypto * From 0a2e4c0f1ec4f8370d483449d76ef90eca4349e8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 14:53:20 -0500 Subject: [PATCH 06/21] remove non-cgo workflow --- .github/workflows/ci.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53141423..0ebd67f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,10 +44,6 @@ jobs: skip-cache: true - name: Run Go Fix run: make go-fix - - name: Run incorrect builds - run: | - echo "::remove-matcher owner=go::" - make incorrect_builds c-code: strategy: From 6c94aa73c885cd49f82ef4c485918b42d8027b58 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 17:07:21 -0500 Subject: [PATCH 07/21] blst v0.3.17 --- blst_src/README.md | 6 +- blst_src/build/cheri/add_mod_256-armv8.S | 22 + blst_src/build/cheri/add_mod_384-armv8.S | 88 +- .../build/cheri/ct_inverse_mod_256-armv8.S | 179 ++-- .../build/cheri/ct_inverse_mod_384-armv8.S | 426 ++++---- .../build/cheri/ct_is_square_mod_384-armv8.S | 22 +- blst_src/build/cheri/div3w-armv8.S | 21 + blst_src/build/cheri/mul_mont_256-armv8.S | 31 +- blst_src/build/cheri/mul_mont_384-armv8.S | 542 +--------- blst_src/build/cheri/sha256-armv8.S | 23 + blst_src/build/coff/add_mod_256-armv8.S | 4 + blst_src/build/coff/add_mod_384-armv8.S | 70 +- .../build/coff/ct_inverse_mod_256-armv8.S | 161 ++- .../build/coff/ct_inverse_mod_384-armv8.S | 398 ++++--- .../build/coff/ct_is_square_mod_384-armv8.S | 4 +- .../build/coff/ctq_inverse_mod_384-x86_64.s | 357 ++++--- .../build/coff/ctx_inverse_mod_384-x86_64.s | 592 ++++++----- blst_src/build/coff/div3w-armv8.S | 3 + blst_src/build/coff/mul_mont_256-armv8.S | 13 +- blst_src/build/coff/mul_mont_384-armv8.S | 528 +--------- blst_src/build/coff/mulq_mont_384-x86_64.s | 803 -------------- blst_src/build/coff/mulx_mont_256-x86_64.s | 34 +- blst_src/build/coff/mulx_mont_384-x86_64.s | 326 +----- blst_src/build/coff/sha256-armv8.S | 5 + blst_src/build/elf/add_mod_256-armv8.S | 22 + blst_src/build/elf/add_mod_384-armv8.S | 88 +- blst_src/build/elf/ct_inverse_mod_256-armv8.S | 179 ++-- blst_src/build/elf/ct_inverse_mod_384-armv8.S | 426 ++++---- .../build/elf/ct_is_square_mod_384-armv8.S | 22 +- .../build/elf/ctq_inverse_mod_384-x86_64.s | 335 +++--- .../build/elf/ctx_inverse_mod_384-x86_64.s | 572 +++++----- blst_src/build/elf/div3w-armv8.S | 21 + blst_src/build/elf/mul_mont_256-armv8.S | 31 +- blst_src/build/elf/mul_mont_384-armv8.S | 542 +--------- blst_src/build/elf/mulq_mont_384-x86_64.s | 762 -------------- blst_src/build/elf/mulx_mont_256-x86_64.s | 34 +- blst_src/build/elf/mulx_mont_384-x86_64.s | 283 +---- blst_src/build/elf/sha256-armv8.S | 23 + blst_src/build/mach-o/add_mod_256-armv8.S | 4 + blst_src/build/mach-o/add_mod_384-armv8.S | 70 +- .../build/mach-o/ct_inverse_mod_256-armv8.S | 161 ++- .../build/mach-o/ct_inverse_mod_384-armv8.S | 392 ++++--- .../build/mach-o/ct_is_square_mod_384-armv8.S | 4 +- .../build/mach-o/ctq_inverse_mod_384-x86_64.s | 319 +++--- .../build/mach-o/ctx_inverse_mod_384-x86_64.s | 552 +++++----- blst_src/build/mach-o/div3w-armv8.S | 3 + blst_src/build/mach-o/mul_mont_256-armv8.S | 13 +- blst_src/build/mach-o/mul_mont_384-armv8.S | 524 +--------- blst_src/build/mach-o/mulq_mont_384-x86_64.s | 762 -------------- blst_src/build/mach-o/mulx_mont_256-x86_64.s | 34 +- blst_src/build/mach-o/mulx_mont_384-x86_64.s | 283 +---- blst_src/build/mach-o/sha256-armv8.S | 5 + blst_src/build/win64/add_mod_256-armv8.asm | 4 + blst_src/build/win64/add_mod_384-armv8.asm | 70 +- blst_src/build/win64/blst.def | 2 + .../build/win64/ct_inverse_mod_256-armv8.asm | 161 ++- .../build/win64/ct_inverse_mod_384-armv8.asm | 382 ++++--- .../win64/ct_is_square_mod_384-armv8.asm | 4 +- .../win64/ctq_inverse_mod_384-x86_64.asm | 361 ++++--- .../win64/ctx_inverse_mod_384-x86_64.asm | 596 ++++++----- blst_src/build/win64/div3w-armv8.asm | 3 + blst_src/build/win64/mul_mont_256-armv8.asm | 13 +- blst_src/build/win64/mul_mont_384-armv8.asm | 524 +--------- blst_src/build/win64/mulq_mont_256-x86_64.asm | 2 + blst_src/build/win64/mulq_mont_384-x86_64.asm | 987 ++---------------- blst_src/build/win64/mulx_mont_256-x86_64.asm | 36 +- blst_src/build/win64/mulx_mont_384-x86_64.asm | 330 +----- blst_src/build/win64/sha256-armv8.asm | 5 + blst_src/cpuid.c | 52 +- blst_src/ec_mult.h | 3 +- blst_src/exports.c | 55 +- blst_src/keygen.c | 2 +- blst_src/multi_scalar.c | 93 +- blst_src/no_asm.h | 52 - blst_src/recip.c | 16 +- blst_src/vect.h | 10 +- internal/blst/blst.go | 137 ++- internal/blst/blst.h | 10 +- internal/blst/blst_aux.h | 3 + 79 files changed, 4876 insertions(+), 10161 deletions(-) diff --git a/blst_src/README.md b/blst_src/README.md index 48abe90e..0bb31ce3 100644 --- a/blst_src/README.md +++ b/blst_src/README.md @@ -1,5 +1,5 @@ All files in this folder contain source files copied from the BLST repo https://github.com/supranational/blst, -specifically from the tagged version `v0.3.14`. +specifically from the tagged version `v0.3.17`. Copyright Supranational LLC Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -18,8 +18,8 @@ The folder contains: - all `/build` (assembly generated files). - this `README` file. -To upgrade the BLST version: -- [ ] audit all BLST updates, with focus on `/src`: https://github.com/supranational/blst/compare/v0.3.14... +Follow these steps from the root of the package to upgrade the BLST version: +- [ ] audit all BLST updates, with focus on `/src`: https://github.com/supranational/blst/compare/v0.3.17... - [ ] delete all files in this folder `./blst_src/` but `blst_src.c` and `README.md`. - [ ] delete all files in `./internal/blst/` but `non_cgo.go`. - [ ] open BLST repository on the new version. diff --git a/blst_src/build/cheri/add_mod_256-armv8.S b/blst_src/build/cheri/add_mod_256-armv8.S index 9db59ce9..21e532c0 100644 --- a/blst_src/build/cheri/add_mod_256-armv8.S +++ b/blst_src/build/cheri/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[c1] ldp x12,x13,[c2] @@ -39,6 +48,7 @@ add_mod_256: .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[c1] ldp x14,x15,[c1,#16] @@ -88,6 +98,7 @@ mul_by_3_mod_256: .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -126,6 +137,7 @@ lshift_mod_256: .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[c1] ldp x10,x11,[c1,#16] @@ -377,3 +389,13 @@ sub_n_check_mod_256: ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/add_mod_384-armv8.S b/blst_src/build/cheri/add_mod_384-armv8.S index 1f498079..d236bbbe 100644 --- a/blst_src/build/cheri/add_mod_384-armv8.S +++ b/blst_src/build/cheri/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ add_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ __add_mod_384_ab_are_loaded: .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ add_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ add_mod_384x: .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ rshift_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ __rshift_mod_384: .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ div_by_2_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ div_by_2_mod_384: .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ lshift_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ __lshift_mod_384: .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ mul_by_3_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ mul_by_3_mod_384: .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ mul_by_8_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ mul_by_8_mod_384: .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ mul_by_3_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ mul_by_3_mod_384x: .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ mul_by_8_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ mul_by_8_mod_384x: .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ cneg_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ cneg_mod_384: .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ sub_mod_384: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ __sub_mod_384: .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ sub_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ sub_mod_384x: .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ mul_by_1_plus_i_mod_384x: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ mul_by_1_plus_i_mod_384x: .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -672,6 +681,7 @@ sgn0_pty_mod_384: .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[c0] ldp x12,x13,[c0,#16] ldp x14,x15,[c0,#32] @@ -753,14 +763,14 @@ sgn0_pty_mod_384x: .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 + ld1 {v0.2d, v1.2d}, [c1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [c2],#48 + ld1 {v3.2d, v4.2d}, [c2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [c0] + st1 {v0.2d, v1.2d}, [c0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ vec_select_32: .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ vec_select_48: .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ vec_select_96: .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ vec_select_192: .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ vec_select_144: .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [c1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ vec_select_288: .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ vec_prefetch: .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [c0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ vec_is_zero_16x: .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [c0], #16 ld1 {v1.2d}, [c1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ vec_is_equal_16x: csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_inverse_mod_256-armv8.S b/blst_src/build/cheri/ct_inverse_mod_256-armv8.S index c451f941..02d12cb6 100644 --- a/blst_src/build/cheri/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/cheri/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-10*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -20,6 +28,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add c1,csp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +48,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[c0,#8*8] // initialize |u| with |f0| @@ -48,18 +57,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add c0,c0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [c0,#8*9] // initialize |v| with |f1| + str x12, [c0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +80,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [c1,#8*8] // |u| - ldr x9, [c1,#8*13] // |v| + ldr x9, [c1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [c0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*5] - stp x5, x5, [c0,#8*7] + stp x4, x5, [c0,#8*4] + stp x5, x5, [c0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [c0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*10] + stp x4, x5, [c0,#8*10] stp x5, x5, [c0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +113,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +138,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +163,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +188,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +213,19 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +238,24 @@ ct_inverse_mod_256: add c0,c0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [c0,#8*4] - + asr x24, x24, #63 + str x24, [c0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [c0,#8*4] - stp x22, x22, [c0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [c0,#8*4] + stp x24, x24, [c0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +270,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +298,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +326,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +354,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +382,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +410,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +438,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [c0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add c0,c0,#8*5 + add c0,c0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +510,7 @@ ct_inverse_mod_256: ldp c23, c24, [c29,#6*__SIZEOF_POINTER__] ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldr c29, [csp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +548,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [c1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [c1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [c1,#8*2+104] + ldp x10, x11, [c1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [c1,#8*4+104] + ldr x23, [c1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +594,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [c1,#8*18] // load rest of |v| + ldr x5, [c1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [c1,#8*20] + ldp x6, x7, [c1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +847,13 @@ __inner_loop_62_256: ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_inverse_mod_384-armv8.S b/blst_src/build/cheri/ct_inverse_mod_384-armv8.S index 8d132974..160e727e 100644 --- a/blst_src/build/cheri/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/cheri/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -22,6 +30,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add c1,csp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +54,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[c0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [c0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [c0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [c1,#8*12] // |u| - ldr x8, [c1,#8*18] // |v| + ldr x8, [c1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +105,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [c0,#8*12] + stp x3, x4, [c0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [c0,#8*14] stp x5, x5, [c0,#8*16] + stp x5, x5, [c0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 + add c0,c0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [c0,#8*6] stp x27, x27, [c0,#8*8] stp x27, x27, [c0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add c0,c0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add c0,c0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +377,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [c0,#8*0] str x9, [c0,#8*6] @@ -375,20 +387,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add c0,c0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [c0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add c0,c0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add c0,c0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [c1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +413,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp c0, c15, [csp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr c30, [c29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [c15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [c15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [c15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [c15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [c0,#8*6] adcs x7, x7, x13 stp x5, x6, [c0,#8*8] @@ -431,15 +480,15 @@ ct_inverse_mod_383: ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [c1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [c1,#8*2+96] @@ -447,6 +496,7 @@ __smul_383x63: ldp x7, x8, [c1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [c1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +511,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [c1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [c1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [c1,#8*2+144] + ldp x11, x12, [c1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [c1,#8*4+144] + ldp x13, x14, [c1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [c1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +552,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +582,41 @@ __smul_383x63: stp x5, x6, [c0,#8*2] adcs x27, x27, x28 stp x7, x27, [c0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [c1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [c1,#8*26] - ldp x7, x8, [c1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [c1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [c1,#8*28] + and x25, x25, x20 + ldp x7, x8, [c1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +624,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +637,15 @@ __smul_767x63_tail: stp x3, x4, [c0,#8*6] adcs x7, x7, x28 stp x5, x6, [c0,#8*8] - adc x8, x8, x28 - stp x7, x8, [c0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [c0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [c1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [c1,#8*2+0] @@ -605,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [c1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [c1,#8*2+48] @@ -643,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +764,7 @@ __smul_383_n_shift_by_62: sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +857,13 @@ __inner_loop_62: ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/ct_is_square_mod_384-armv8.S b/blst_src/build/cheri/ct_is_square_mod_384-armv8.S index 269c300a..183a982e 100644 --- a/blst_src/build/cheri/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/cheri/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp c29, c30, [csp,#-16*__SIZEOF_POINTER__]! add c29, csp, #0 stp c19, c20, [csp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ct_is_square_mod_384: ldp c25, c26, [c29,#8*__SIZEOF_POINTER__] ldp c27, c28, [c29,#10*__SIZEOF_POINTER__] ldr c29, [csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ __inner_loop_48: ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/div3w-armv8.S b/blst_src/build/cheri/div3w-armv8.S index 64149096..8e5cd764 100644 --- a/blst_src/build/cheri/div3w-armv8.S +++ b/blst_src/build/cheri/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[c0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ div_3_limbs: .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[c1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ quot_rem_128: .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[c1] ldr x8,[c0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ quot_rem_64: ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/mul_mont_256-armv8.S b/blst_src/build/cheri/mul_mont_256-armv8.S index 2dd8f459..3c94f7dc 100644 --- a/blst_src/build/cheri/mul_mont_256-armv8.S +++ b/blst_src/build/cheri/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ mul_mont_sparse_256: .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-6*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ sqr_mont_sparse_256: ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldr c29,[csp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ sqr_mont_sparse_256: .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -330,7 +339,7 @@ from_mont_256: stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ from_mont_256: .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -374,7 +383,7 @@ redc_mont_256: stp x12,x13,[c0,#16] ldr c29,[csp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ __mul_by_1_mont_256: ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/mul_mont_384-armv8.S b/blst_src/build/cheri/mul_mont_384-armv8.S index 28e669e6..b475ed61 100644 --- a/blst_src/build/cheri/mul_mont_384-armv8.S +++ b/blst_src/build/cheri/mul_mont_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 @@ -5,7 +13,7 @@ .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -23,7 +31,7 @@ add_mod_384x384: ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -87,7 +95,7 @@ __add_mod_384x384: .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-8*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -105,7 +113,7 @@ sub_mod_384x384: ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] ldr c29,[csp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -241,7 +249,7 @@ __sub_mod_384: .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -314,7 +322,7 @@ mul_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -323,7 +331,7 @@ mul_mont_384x: .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -399,7 +407,7 @@ sqr_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -408,7 +416,7 @@ sqr_mont_384x: .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -440,7 +448,7 @@ mul_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -821,7 +829,7 @@ __mul_mont_384: .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -858,7 +866,7 @@ sqr_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -867,7 +875,7 @@ sqr_mont_384: .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -923,7 +931,7 @@ sqr_n_mul_mont_383: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1044,7 +1052,7 @@ __sqr_384: .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1074,7 @@ sqr_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1075,7 +1083,7 @@ sqr_384: .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1107,7 @@ redc_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1108,7 +1116,7 @@ redc_mont_384: .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1157,7 @@ from_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1367,7 +1375,7 @@ __redc_tail_mont_384: .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1393,7 @@ mul_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1571,7 +1579,7 @@ __mul_384: .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1661,7 @@ mul_382x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1662,7 +1670,7 @@ mul_382x: .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -1753,480 +1761,16 @@ sqr_382x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,%function -.align 5 -sqr_mont_382x: - .inst 0xd503233f - stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! - add c29,csp,#0 - stp c19,c20,[csp,#2*__SIZEOF_POINTER__] - stp c21,c22,[csp,#4*__SIZEOF_POINTER__] - stp c23,c24,[csp,#6*__SIZEOF_POINTER__] - stp c25,c26,[csp,#8*__SIZEOF_POINTER__] - stp c27,c28,[csp,#10*__SIZEOF_POINTER__] - stp c3,c0,[csp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub csp,csp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[c1] - ldp x13,x14,[c1,#16] - ldp x15,x16,[c1,#32] - - ldp x17,x20,[c1,#48] - ldp x21,x22,[c1,#64] - ldp x23,x24,[c1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[csp] - stp x7,x8,[csp,#16] - stp x9,x10,[csp,#32] - stp x19,x20,[csp,#48] - stp x21,x22,[csp,#64] - stp x23,x24,[csp,#80] - str x25,[csp,#96] - - ldp x5,x6,[c2] - ldp x7,x8,[c2,#16] - ldp x9,x10,[c2,#32] - - add c2,c1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[c2,#48] - stp x21,x22,[c2,#64] - stp x23,x24,[c2,#80] - - ldp x11,x12,[csp] - ldr x17,[csp,#48] - ldp x13,x14,[csp,#16] - ldp x15,x16,[csp,#32] - - add c2,csp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr c30,[c29,#__SIZEOF_POINTER__] - - ldr x25,[csp,#96] // account for sign from a->re - a->im - ldp x19,x20,[csp] - ldp x21,x22,[csp,#16] - ldp x23,x24,[csp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[c2] - stp x13,x14,[c2,#16] - stp x15,x16,[c2,#32] - - add csp,csp,#112 - ldp c19,c20,[c29,#2*__SIZEOF_POINTER__] - ldp c21,c22,[c29,#4*__SIZEOF_POINTER__] - ldp c23,c24,[c29,#6*__SIZEOF_POINTER__] - ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] - ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] - ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf - ret -.size sqr_mont_382x,.-sqr_mont_382x - -.type __mul_mont_383_nonred,%function -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[c2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[c2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[c29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp c4,c2,[c29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - - ret -.size __mul_mont_383_nonred,.-__mul_mont_383_nonred - .globl sgn0_pty_mont_384 .hidden sgn0_pty_mont_384 .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1815,7 @@ sgn0_pty_mont_384: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2280,7 +1824,7 @@ sgn0_pty_mont_384: .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp c29,c30,[csp,#-16*__SIZEOF_POINTER__]! add c29,csp,#0 stp c19,c20,[csp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1913,16 @@ sgn0_pty_mont_384x: ldp c25,c26,[c29,#8*__SIZEOF_POINTER__] ldp c27,c28,[c29,#10*__SIZEOF_POINTER__] ldr c29,[csp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/cheri/sha256-armv8.S b/blst_src/build/cheri/sha256-armv8.S index 53d011ec..7b018952 100644 --- a/blst_src/build/cheri/sha256-armv8.S +++ b/blst_src/build/cheri/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -47,6 +55,7 @@ .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp c29,c30,[csp,#-2*__SIZEOF_POINTER__]! add c29,csp,#0 @@ -187,6 +196,7 @@ blst_sha256_block_armv8: .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp c16,__blst_platform_cap ldr w16,[c16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1036,6 +1046,7 @@ blst_sha256_block_data_order: .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1075,7 @@ blst_sha256_emit: .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[c1],#1 sub x2,x2,#1 @@ -1077,9 +1089,20 @@ blst_sha256_bcopy: .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[c1] ldp x6,x7,[c1,#16] stp x4,x5,[c0] stp x6,x7,[c0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/coff/add_mod_256-armv8.S b/blst_src/build/coff/add_mod_256-armv8.S index 27b64ef4..07495153 100644 --- a/blst_src/build/coff/add_mod_256-armv8.S +++ b/blst_src/build/coff/add_mod_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -43,6 +44,7 @@ add_mod_256: .endef .p2align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -94,6 +96,7 @@ mul_by_3_mod_256: .endef .p2align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -134,6 +137,7 @@ lshift_mod_256: .endef .p2align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/coff/add_mod_384-armv8.S b/blst_src/build/coff/add_mod_384-armv8.S index c1709000..65275894 100644 --- a/blst_src/build/coff/add_mod_384-armv8.S +++ b/blst_src/build/coff/add_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -27,7 +27,7 @@ add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -77,7 +77,7 @@ __add_mod_384_ab_are_loaded: .endef .p2align 5 add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +105,7 @@ add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -116,7 +116,7 @@ add_mod_384x: .endef .p2align 5 rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -143,7 +143,7 @@ rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -182,7 +182,7 @@ __rshift_mod_384: .endef .p2align 5 div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -206,7 +206,7 @@ div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -217,7 +217,7 @@ div_by_2_mod_384: .endef .p2align 5 lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -244,7 +244,7 @@ lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -286,7 +286,7 @@ __lshift_mod_384: .endef .p2align 5 mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -316,7 +316,7 @@ mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -327,7 +327,7 @@ mul_by_3_mod_384: .endef .p2align 5 mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -353,7 +353,7 @@ mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -364,7 +364,7 @@ mul_by_8_mod_384: .endef .p2align 5 mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -409,7 +409,7 @@ mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -420,7 +420,7 @@ mul_by_3_mod_384x: .endef .p2align 5 mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -457,7 +457,7 @@ mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -468,7 +468,7 @@ mul_by_8_mod_384x: .endef .p2align 5 cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -510,7 +510,7 @@ cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -521,7 +521,7 @@ cneg_mod_384: .endef .p2align 5 sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -541,7 +541,7 @@ sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -588,7 +588,7 @@ __sub_mod_384: .endef .p2align 5 sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -616,7 +616,7 @@ sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -627,7 +627,7 @@ sub_mod_384x: .endef .p2align 5 mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -660,7 +660,7 @@ mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -671,6 +671,7 @@ mul_by_1_plus_i_mod_384x: .endef .p2align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -710,6 +711,7 @@ sgn0_pty_mod_384: .endef .p2align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -793,14 +795,14 @@ sgn0_pty_mod_384x: .endef .p2align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl vec_select_48 @@ -810,6 +812,7 @@ vec_select_32: .endef .p2align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -827,6 +830,7 @@ vec_select_48: .endef .p2align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -850,6 +854,7 @@ vec_select_96: .endef .p2align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -885,6 +890,7 @@ vec_select_192: .endef .p2align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -914,6 +920,7 @@ vec_select_144: .endef .p2align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -961,6 +968,7 @@ vec_select_288: .endef .p2align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -1003,6 +1011,7 @@ vec_prefetch: .endef .p2align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -1030,6 +1039,7 @@ vec_is_zero_16x: .endef .p2align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/coff/ct_inverse_mod_256-armv8.S b/blst_src/build/coff/ct_inverse_mod_256-armv8.S index b8ac9bf7..90d01408 100644 --- a/blst_src/build/coff/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/coff/ct_inverse_mod_256-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -41,7 +42,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -50,18 +51,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -73,29 +74,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -108,25 +107,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -139,25 +132,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -170,25 +157,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -201,25 +182,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -232,25 +207,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -263,25 +232,24 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -296,21 +264,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -325,21 +292,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -354,21 +320,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -383,21 +348,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -412,21 +376,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -441,21 +404,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -470,16 +432,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -543,7 +504,7 @@ ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -583,11 +544,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -631,9 +592,9 @@ __smul_256x63: .p2align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/blst_src/build/coff/ct_inverse_mod_384-armv8.S b/blst_src/build/coff/ct_inverse_mod_384-armv8.S index aecb4ea7..4a94a0a8 100644 --- a/blst_src/build/coff/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/coff/ct_inverse_mod_384-armv8.S @@ -1,13 +1,13 @@ .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; +.def ct_inverse_mod_384; .type 32; .endef .p2align 5 -ct_inverse_mod_383: -.long 3573752639 +ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -24,6 +24,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -47,40 +48,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -98,266 +99,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -367,7 +371,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -377,20 +381,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -401,25 +407,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -433,17 +474,17 @@ ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.def __smul_383x63; +.def __smul_384x63; .type 32; .endef .p2align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -451,6 +492,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -465,28 +507,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -501,21 +548,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -527,43 +578,43 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.def __smul_767x63_tail; +.def __smul_768x63_tail; .type 32; .endef .p2align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -571,10 +622,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -583,17 +635,17 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.def __smul_383_n_shift_by_62; +.def __smul_384_n_shift_by_62; .type 32; .endef .p2align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -613,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -651,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/coff/ct_is_square_mod_384-armv8.S b/blst_src/build/coff/ct_is_square_mod_384-armv8.S index 07f3d9f0..ccf81578 100644 --- a/blst_src/build/coff/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/coff/ct_is_square_mod_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -86,7 +86,7 @@ ct_is_square_mod_384: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s b/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s index d00335b9..b86dada5 100644 --- a/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/coff/ctq_inverse_mod_384-x86_64.s @@ -1,16 +1,16 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 +.globl ct_inverse_mod_384 -.def ct_inverse_mod_383; .scl 2; .type 32; .endef +.def ct_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ct_inverse_mod_383: +ct_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ct_inverse_mod_383: +.LSEH_begin_ct_inverse_mod_384: movq %rcx,%rdi @@ -19,7 +19,7 @@ ct_inverse_mod_383: movq %r9,%rcx #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp @@ -35,7 +35,7 @@ ct_inverse_mod_383: subq $1112,%rsp -.LSEH_body_ct_inverse_mod_383: +.LSEH_body_ct_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -82,7 +82,7 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -90,10 +90,10 @@ ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -106,19 +106,19 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -135,6 +135,7 @@ ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -145,13 +146,14 @@ ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -162,14 +164,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -177,12 +179,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -193,14 +195,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -208,12 +210,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -224,14 +226,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -239,12 +241,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -255,14 +257,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -270,19 +272,17 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -293,14 +293,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -308,12 +308,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -324,14 +324,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -339,12 +339,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -355,14 +355,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -370,12 +370,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -386,14 +386,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -401,12 +401,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -417,14 +417,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -432,12 +432,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -461,16 +461,16 @@ ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -493,37 +493,77 @@ ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -540,7 +580,7 @@ ct_inverse_mod_383: leaq 48(%r8),%rsp -.LSEH_epilogue_ct_inverse_mod_383: +.LSEH_epilogue_ct_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -554,10 +594,10 @@ ct_inverse_mod_383: .byte 0xf3,0xc3 #endif -.LSEH_end_ct_inverse_mod_383: -.def __smulq_767x63; .scl 3; .type 32; .endef +.LSEH_end_ct_inverse_mod_384: +.def __smulq_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_767x63: +__smulq_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -566,6 +606,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -574,7 +615,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -585,16 +626,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -620,14 +665,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -730,39 +775,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -774,9 +821,9 @@ __smulq_767x63: .byte 0xf3,0xc3 #endif -.def __smulq_383x63; .scl 3; .type 32; .endef +.def __smulq_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulq_383x63: +__smulq_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -785,6 +832,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -800,16 +848,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -831,10 +883,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -842,13 +895,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -864,16 +919,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -895,17 +954,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -913,6 +974,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -924,9 +986,9 @@ __smulq_383x63: .byte 0xf3,0xc3 #endif -.def __smulq_383_n_shift_by_62; .scl 3; .type 32; .endef +.def __smulq_384_n_shift_by_62; .scl 3; .type 32; .endef .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1236,28 +1303,28 @@ __inner_loop_62: .section .pdata .p2align 2 -.rva .LSEH_begin_ct_inverse_mod_383 -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_prologue +.rva .LSEH_begin_ct_inverse_mod_384 +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_prologue -.rva .LSEH_body_ct_inverse_mod_383 -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_body +.rva .LSEH_body_ct_inverse_mod_384 +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_body -.rva .LSEH_epilogue_ct_inverse_mod_383 -.rva .LSEH_end_ct_inverse_mod_383 -.rva .LSEH_info_ct_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ct_inverse_mod_384 +.rva .LSEH_end_ct_inverse_mod_384 +.rva .LSEH_info_ct_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ct_inverse_mod_383_prologue: +.LSEH_info_ct_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ct_inverse_mod_383_body: +.LSEH_info_ct_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1270,7 +1337,7 @@ __inner_loop_62: .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ct_inverse_mod_383_epilogue: +.LSEH_info_ct_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s b/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s index ed3002a4..060940a5 100644 --- a/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/coff/ctx_inverse_mod_384-x86_64.s @@ -1,22 +1,22 @@ .text -.globl ctx_inverse_mod_383 +.globl ctx_inverse_mod_384 -.def ctx_inverse_mod_383; .scl 2; .type 32; .endef +.def ctx_inverse_mod_384; .scl 2; .type 32; .endef .p2align 5 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .byte 0xf3,0x0f,0x1e,0xfa movq %rdi,8(%rsp) movq %rsi,16(%rsp) movq %rsp,%r11 -.LSEH_begin_ctx_inverse_mod_383: +.LSEH_begin_ctx_inverse_mod_384: movq %rcx,%rdi movq %rdx,%rsi movq %r8,%rdx movq %r9,%rcx -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp pushq %rbx @@ -31,7 +31,7 @@ ct_inverse_mod_383$1: subq $1112,%rsp -.LSEH_body_ctx_inverse_mod_383: +.LSEH_body_ctx_inverse_mod_384: leaq 88+511(%rsp),%rax @@ -81,7 +81,7 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -89,10 +89,10 @@ ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -105,19 +105,19 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -134,6 +134,7 @@ ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -144,13 +145,14 @@ ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -161,14 +163,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -176,12 +178,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -192,14 +194,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -207,12 +209,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -223,14 +225,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -238,12 +240,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -254,14 +256,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -269,12 +271,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -285,14 +287,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -300,12 +302,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -316,14 +318,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -331,12 +333,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -347,14 +349,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -362,12 +364,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -378,14 +380,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -393,12 +395,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -409,14 +411,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -424,12 +426,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -440,14 +442,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -455,19 +457,17 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -478,14 +478,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -493,12 +493,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -509,14 +509,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -524,12 +524,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -540,14 +540,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -555,12 +555,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -571,14 +571,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -586,12 +586,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -602,14 +602,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -617,12 +617,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -633,14 +633,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -648,12 +648,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -664,14 +664,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -679,12 +679,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -710,12 +710,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -741,12 +741,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -772,12 +772,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -803,21 +803,21 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -834,40 +834,80 @@ ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -884,7 +924,7 @@ ct_inverse_mod_383$1: leaq 48(%r8),%rsp -.LSEH_epilogue_ctx_inverse_mod_383: +.LSEH_epilogue_ctx_inverse_mod_384: mov 8(%rsp),%rdi mov 16(%rsp),%rsi @@ -898,10 +938,10 @@ ct_inverse_mod_383$1: .byte 0xf3,0xc3 #endif -.LSEH_end_ctx_inverse_mod_383: -.def __smulx_767x63; .scl 3; .type 32; .endef +.LSEH_end_ctx_inverse_mod_384: +.def __smulx_768x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_767x63: +__smulx_768x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0(%rsi),%r8 @@ -910,6 +950,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -918,7 +959,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -928,37 +969,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -993,7 +1038,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1005,62 +1050,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1072,9 +1119,9 @@ __smulx_767x63: .byte 0xf3,0xc3 #endif -.def __smulx_383x63; .scl 3; .type 32; .endef +.def __smulx_384x63; .scl 3; .type 32; .endef .p2align 5 -__smulx_383x63: +__smulx_384x63: .byte 0xf3,0x0f,0x1e,0xfa movq 0+0(%rsi),%r8 @@ -1083,6 +1130,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1098,12 +1146,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1117,19 +1170,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1145,12 +1201,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1163,13 +1224,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1177,6 +1240,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1188,13 +1252,12 @@ __smulx_383x63: .byte 0xf3,0xc3 #endif -.def __smulx_383_n_shift_by_31; .scl 3; .type 32; .endef +.def __smulx_384_n_shift_by_31; .scl 3; .type 32; .endef .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1578,9 +1646,9 @@ __inner_loop_31: #endif -.def __tail_loop_53; .scl 3; .type 32; .endef +.def __tail_loop_55; .scl 3; .type 32; .endef .p2align 5 -__tail_loop_53: +__tail_loop_55: .byte 0xf3,0x0f,0x1e,0xfa movq $1,%rdx @@ -1588,7 +1656,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1615,7 +1683,7 @@ __tail_loop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1629,28 +1697,28 @@ __tail_loop_53: .section .pdata .p2align 2 -.rva .LSEH_begin_ctx_inverse_mod_383 -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_prologue +.rva .LSEH_begin_ctx_inverse_mod_384 +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_prologue -.rva .LSEH_body_ctx_inverse_mod_383 -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_body +.rva .LSEH_body_ctx_inverse_mod_384 +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_body -.rva .LSEH_epilogue_ctx_inverse_mod_383 -.rva .LSEH_end_ctx_inverse_mod_383 -.rva .LSEH_info_ctx_inverse_mod_383_epilogue +.rva .LSEH_epilogue_ctx_inverse_mod_384 +.rva .LSEH_end_ctx_inverse_mod_384 +.rva .LSEH_info_ctx_inverse_mod_384_epilogue .section .xdata .p2align 3 -.LSEH_info_ctx_inverse_mod_383_prologue: +.LSEH_info_ctx_inverse_mod_384_prologue: .byte 1,0,5,0x0b .byte 0,0x74,1,0 .byte 0,0x64,2,0 .byte 0,0xb3 .byte 0,0 .long 0,0 -.LSEH_info_ctx_inverse_mod_383_body: +.LSEH_info_ctx_inverse_mod_384_body: .byte 1,0,18,0 .byte 0x00,0xf4,0x8b,0x00 .byte 0x00,0xe4,0x8c,0x00 @@ -1663,7 +1731,7 @@ __tail_loop_53: .byte 0x00,0x01,0x91,0x00 .byte 0x00,0x00,0x00,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_ctx_inverse_mod_383_epilogue: +.LSEH_info_ctx_inverse_mod_384_epilogue: .byte 1,0,4,0 .byte 0x00,0x74,0x01,0x00 .byte 0x00,0x64,0x02,0x00 diff --git a/blst_src/build/coff/div3w-armv8.S b/blst_src/build/coff/div3w-armv8.S index fd694935..eaa741f4 100644 --- a/blst_src/build/coff/div3w-armv8.S +++ b/blst_src/build/coff/div3w-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -43,6 +44,7 @@ div_3_limbs: .endef .p2align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -82,6 +84,7 @@ quot_rem_128: .endef .p2align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/blst_src/build/coff/mul_mont_256-armv8.S b/blst_src/build/coff/mul_mont_256-armv8.S index 1b256a8a..ce49bae3 100644 --- a/blst_src/build/coff/mul_mont_256-armv8.S +++ b/blst_src/build/coff/mul_mont_256-armv8.S @@ -7,6 +7,7 @@ .endef .p2align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -200,7 +201,7 @@ mul_mont_sparse_256: .endef .p2align 5 sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -301,7 +302,7 @@ sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl from_mont_256 @@ -311,7 +312,7 @@ sqr_mont_sparse_256: .endef .p2align 5 from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -336,7 +337,7 @@ from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -347,7 +348,7 @@ from_mont_256: .endef .p2align 5 redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -382,7 +383,7 @@ redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/mul_mont_384-armv8.S b/blst_src/build/coff/mul_mont_384-armv8.S index a5e1cfef..79896c2b 100644 --- a/blst_src/build/coff/mul_mont_384-armv8.S +++ b/blst_src/build/coff/mul_mont_384-armv8.S @@ -7,7 +7,7 @@ .endef .p2align 5 add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -93,7 +93,7 @@ __add_mod_384x384: .endef .p2align 5 sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -111,7 +111,7 @@ sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -255,7 +255,7 @@ __sub_mod_384: .endef .p2align 5 mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -328,7 +328,7 @@ mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -339,7 +339,7 @@ mul_mont_384x: .endef .p2align 5 sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -415,7 +415,7 @@ sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -426,7 +426,7 @@ sqr_mont_384x: .endef .p2align 5 mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -458,7 +458,7 @@ mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -843,7 +843,7 @@ __mul_mont_384: .endef .p2align 5 sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -880,7 +880,7 @@ sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -891,7 +891,7 @@ sqr_mont_384: .endef .p2align 5 sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -947,7 +947,7 @@ sqr_n_mul_mont_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .def __sqr_384; @@ -1072,7 +1072,7 @@ __sqr_384: .endef .p2align 5 sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1094,7 +1094,7 @@ sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1105,7 +1105,7 @@ sqr_384: .endef .p2align 5 redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1129,7 +1129,7 @@ redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1140,7 +1140,7 @@ redc_mont_384: .endef .p2align 5 from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1181,7 +1181,7 @@ from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1405,7 +1405,7 @@ __redc_tail_mont_384: .endef .p2align 5 mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1423,7 +1423,7 @@ mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1613,7 +1613,7 @@ __mul_384: .endef .p2align 5 mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1695,7 +1695,7 @@ mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1706,7 +1706,7 @@ mul_382x: .endef .p2align 5 sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1797,475 +1797,7 @@ sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.globl sqr_mont_382x - -.def sqr_mont_382x; -.type 32; -.endef -.p2align 5 -sqr_mont_382x: -.long 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.def __mul_mont_383_nonred; -.type 32; -.endef -.p2align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret @@ -2276,7 +1808,7 @@ __mul_mont_383_nonred: .endef .p2align 5 sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2321,7 +1853,7 @@ sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2332,7 +1864,7 @@ sgn0_pty_mont_384: .endef .p2align 5 sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2421,6 +1953,6 @@ sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/coff/mulq_mont_384-x86_64.s b/blst_src/build/coff/mulq_mont_384-x86_64.s index 6434f46d..c214a906 100644 --- a/blst_src/build/coff/mulq_mont_384-x86_64.s +++ b/blst_src/build/coff/mulq_mont_384-x86_64.s @@ -3158,771 +3158,6 @@ sqr_n_mul_mont_383: #endif .LSEH_end_sqr_n_mul_mont_383: -.def __mulq_mont_383_nonred; .scl 3; .type 32; .endef -.p2align 5 -__mulq_mont_383_nonred: - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.globl sqr_mont_382x - -.def sqr_mont_382x; .scl 2; .type 32; .endef -.p2align 5 -sqr_mont_382x: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_sqr_mont_382x: - - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx -#ifdef __BLST_PORTABLE__ - testl $1,__blst_platform_cap(%rip) - jnz sqr_mont_382x$1 -#endif - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $136,%rsp - -.LSEH_body_sqr_mont_382x: - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 - - movq 8(%r8),%r14 - - movq 16(%r8),%r13 - - movq 24(%r8),%r12 - - movq 32(%r8),%rbx - - movq 40(%r8),%rbp - - leaq 48(%r8),%rsp - -.LSEH_epilogue_sqr_mont_382x: - mov 8(%rsp),%rdi - mov 16(%rsp),%rsi - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.LSEH_end_sqr_mont_382x: .section .pdata .p2align 2 .rva .LSEH_begin_mul_mont_384x @@ -4093,18 +3328,6 @@ sqr_mont_382x: .rva .LSEH_end_sqr_n_mul_mont_383 .rva .LSEH_info_sqr_n_mul_mont_383_epilogue -.rva .LSEH_begin_sqr_mont_382x -.rva .LSEH_body_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_prologue - -.rva .LSEH_body_sqr_mont_382x -.rva .LSEH_epilogue_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_body - -.rva .LSEH_epilogue_sqr_mont_382x -.rva .LSEH_end_sqr_mont_382x -.rva .LSEH_info_sqr_mont_382x_epilogue - .section .xdata .p2align 3 .LSEH_info_mul_mont_384x_prologue: @@ -4467,29 +3690,3 @@ sqr_mont_382x: .byte 0x00,0x64,0x02,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqr_mont_382x_prologue: -.byte 1,0,5,0x0b -.byte 0,0x74,1,0 -.byte 0,0x64,2,0 -.byte 0,0xb3 -.byte 0,0 -.long 0,0 -.LSEH_info_sqr_mont_382x_body: -.byte 1,0,18,0 -.byte 0x00,0xf4,0x11,0x00 -.byte 0x00,0xe4,0x12,0x00 -.byte 0x00,0xd4,0x13,0x00 -.byte 0x00,0xc4,0x14,0x00 -.byte 0x00,0x34,0x15,0x00 -.byte 0x00,0x54,0x16,0x00 -.byte 0x00,0x74,0x18,0x00 -.byte 0x00,0x64,0x19,0x00 -.byte 0x00,0x01,0x17,0x00 -.byte 0x00,0x00,0x00,0x00 -.byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqr_mont_382x_epilogue: -.byte 1,0,4,0 -.byte 0x00,0x74,0x01,0x00 -.byte 0x00,0x64,0x02,0x00 -.byte 0x00,0x00,0x00,0x00 - diff --git a/blst_src/build/coff/mulx_mont_256-x86_64.s b/blst_src/build/coff/mulx_mont_256-x86_64.s index 307d9185..1ecc9a73 100644 --- a/blst_src/build/coff/mulx_mont_256-x86_64.s +++ b/blst_src/build/coff/mulx_mont_256-x86_64.s @@ -212,17 +212,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -257,17 +254,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -302,16 +296,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -328,11 +319,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/coff/mulx_mont_384-x86_64.s b/blst_src/build/coff/mulx_mont_384-x86_64.s index 69fcbd73..0e1c2a4d 100644 --- a/blst_src/build/coff/mulx_mont_384-x86_64.s +++ b/blst_src/build/coff/mulx_mont_384-x86_64.s @@ -2064,8 +2064,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2093,11 +2093,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2127,8 +2125,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2156,11 +2154,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2190,8 +2186,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2219,11 +2215,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2253,8 +2247,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2282,11 +2276,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2316,8 +2308,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2345,11 +2337,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2380,10 +2370,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2789,9 +2778,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2849,9 +2837,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2909,9 +2896,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2969,9 +2955,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -3029,9 +3014,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3085,214 +3069,6 @@ __mulx_mont_383_nonred: #endif -.globl sqrx_mont_382x - -.def sqrx_mont_382x; .scl 2; .type 32; .endef -.p2align 5 -sqrx_mont_382x: - .byte 0xf3,0x0f,0x1e,0xfa - movq %rdi,8(%rsp) - movq %rsi,16(%rsp) - movq %rsp,%r11 -.LSEH_begin_sqrx_mont_382x: - - - movq %rcx,%rdi - movq %rdx,%rsi - movq %r8,%rdx - movq %r9,%rcx -sqr_mont_382x$1: - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $136,%rsp - -.LSEH_body_sqrx_mont_382x: - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 - - movq 8(%r8),%r14 - - movq 16(%r8),%r13 - - movq 24(%r8),%r12 - - movq 32(%r8),%rbx - - movq 40(%r8),%rbp - - leaq 48(%r8),%rsp - -.LSEH_epilogue_sqrx_mont_382x: - mov 8(%rsp),%rdi - mov 16(%rsp),%rsi - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif - -.LSEH_end_sqrx_mont_382x: .section .pdata .p2align 2 .rva .LSEH_begin_mulx_mont_384x @@ -3463,18 +3239,6 @@ sqr_mont_382x$1: .rva .LSEH_end_sqrx_n_mul_mont_383 .rva .LSEH_info_sqrx_n_mul_mont_383_epilogue -.rva .LSEH_begin_sqrx_mont_382x -.rva .LSEH_body_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_prologue - -.rva .LSEH_body_sqrx_mont_382x -.rva .LSEH_epilogue_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_body - -.rva .LSEH_epilogue_sqrx_mont_382x -.rva .LSEH_end_sqrx_mont_382x -.rva .LSEH_info_sqrx_mont_382x_epilogue - .section .xdata .p2align 3 .LSEH_info_mulx_mont_384x_prologue: @@ -3841,29 +3605,3 @@ sqr_mont_382x$1: .byte 0x00,0x64,0x02,0x00 .byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqrx_mont_382x_prologue: -.byte 1,0,5,0x0b -.byte 0,0x74,1,0 -.byte 0,0x64,2,0 -.byte 0,0xb3 -.byte 0,0 -.long 0,0 -.LSEH_info_sqrx_mont_382x_body: -.byte 1,0,18,0 -.byte 0x00,0xf4,0x11,0x00 -.byte 0x00,0xe4,0x12,0x00 -.byte 0x00,0xd4,0x13,0x00 -.byte 0x00,0xc4,0x14,0x00 -.byte 0x00,0x34,0x15,0x00 -.byte 0x00,0x54,0x16,0x00 -.byte 0x00,0x74,0x18,0x00 -.byte 0x00,0x64,0x19,0x00 -.byte 0x00,0x01,0x17,0x00 -.byte 0x00,0x00,0x00,0x00 -.byte 0x00,0x00,0x00,0x00 -.LSEH_info_sqrx_mont_382x_epilogue: -.byte 1,0,4,0 -.byte 0x00,0x74,0x01,0x00 -.byte 0x00,0x64,0x02,0x00 -.byte 0x00,0x00,0x00,0x00 - diff --git a/blst_src/build/coff/sha256-armv8.S b/blst_src/build/coff/sha256-armv8.S index dcf1b5e7..ee5af87d 100644 --- a/blst_src/build/coff/sha256-armv8.S +++ b/blst_src/build/coff/sha256-armv8.S @@ -49,6 +49,7 @@ .endef .p2align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -191,6 +192,7 @@ blst_sha256_block_armv8: .endef .p2align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1042,6 +1044,7 @@ blst_sha256_block_data_order: .endef .p2align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1072,6 +1075,7 @@ blst_sha256_emit: .endef .p2align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1087,6 +1091,7 @@ blst_sha256_bcopy: .endef .p2align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/build/elf/add_mod_256-armv8.S b/blst_src/build/elf/add_mod_256-armv8.S index 57476aaa..4d1d791b 100644 --- a/blst_src/build/elf/add_mod_256-armv8.S +++ b/blst_src/build/elf/add_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_256 @@ -5,6 +13,7 @@ .type add_mod_256,%function .align 5 add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +48,7 @@ add_mod_256: .type mul_by_3_mod_256,%function .align 5 mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +98,7 @@ mul_by_3_mod_256: .type lshift_mod_256,%function .align 5 lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +137,7 @@ lshift_mod_256: .type rshift_mod_256,%function .align 5 rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -377,3 +389,13 @@ sub_n_check_mod_256: ret .size sub_n_check_mod_256,.-sub_n_check_mod_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/add_mod_384-armv8.S b/blst_src/build/elf/add_mod_384-armv8.S index f5c0ead0..da6ee40a 100644 --- a/blst_src/build/elf/add_mod_384-armv8.S +++ b/blst_src/build/elf/add_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384 @@ -5,7 +13,7 @@ .type add_mod_384,%function .align 5 add_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +33,7 @@ add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384,.-add_mod_384 @@ -71,7 +79,7 @@ __add_mod_384_ab_are_loaded: .type add_mod_384x,%function .align 5 add_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +107,7 @@ add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x,.-add_mod_384x @@ -108,7 +116,7 @@ add_mod_384x: .type rshift_mod_384,%function .align 5 rshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +143,7 @@ rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size rshift_mod_384,.-rshift_mod_384 @@ -170,7 +178,7 @@ __rshift_mod_384: .type div_by_2_mod_384,%function .align 5 div_by_2_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +202,7 @@ div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size div_by_2_mod_384,.-div_by_2_mod_384 @@ -203,7 +211,7 @@ div_by_2_mod_384: .type lshift_mod_384,%function .align 5 lshift_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +238,7 @@ lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size lshift_mod_384,.-lshift_mod_384 @@ -268,7 +276,7 @@ __lshift_mod_384: .type mul_by_3_mod_384,%function .align 5 mul_by_3_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +306,7 @@ mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384,.-mul_by_3_mod_384 @@ -307,7 +315,7 @@ mul_by_3_mod_384: .type mul_by_8_mod_384,%function .align 5 mul_by_8_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +341,7 @@ mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384,.-mul_by_8_mod_384 @@ -342,7 +350,7 @@ mul_by_8_mod_384: .type mul_by_3_mod_384x,%function .align 5 mul_by_3_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +395,7 @@ mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_3_mod_384x,.-mul_by_3_mod_384x @@ -396,7 +404,7 @@ mul_by_3_mod_384x: .type mul_by_8_mod_384x,%function .align 5 mul_by_8_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +441,7 @@ mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_8_mod_384x,.-mul_by_8_mod_384x @@ -442,7 +450,7 @@ mul_by_8_mod_384x: .type cneg_mod_384,%function .align 5 cneg_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +492,7 @@ cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size cneg_mod_384,.-cneg_mod_384 @@ -493,7 +501,7 @@ cneg_mod_384: .type sub_mod_384,%function .align 5 sub_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +521,7 @@ sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384,.-sub_mod_384 @@ -556,7 +564,7 @@ __sub_mod_384: .type sub_mod_384x,%function .align 5 sub_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +592,7 @@ sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x,.-sub_mod_384x @@ -593,7 +601,7 @@ sub_mod_384x: .type mul_by_1_plus_i_mod_384x,%function .align 5 mul_by_1_plus_i_mod_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +634,7 @@ mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_by_1_plus_i_mod_384x,.-mul_by_1_plus_i_mod_384x @@ -635,6 +643,7 @@ mul_by_1_plus_i_mod_384x: .type sgn0_pty_mod_384,%function .align 5 sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +681,7 @@ sgn0_pty_mod_384: .type sgn0_pty_mod_384x,%function .align 5 sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +763,14 @@ sgn0_pty_mod_384x: .type vec_select_32,%function .align 5 vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .size vec_select_32,.-vec_select_32 .globl vec_select_48 @@ -768,6 +778,7 @@ vec_select_32: .type vec_select_48,%function .align 5 vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +794,7 @@ vec_select_48: .type vec_select_96,%function .align 5 vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +816,7 @@ vec_select_96: .type vec_select_192,%function .align 5 vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +850,7 @@ vec_select_192: .type vec_select_144,%function .align 5 vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +878,7 @@ vec_select_144: .type vec_select_288,%function .align 5 vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +924,7 @@ vec_select_288: .type vec_prefetch,%function .align 5 vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +965,7 @@ vec_prefetch: .type vec_is_zero_16x,%function .align 5 vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +991,7 @@ vec_is_zero_16x: .type vec_is_equal_16x,%function .align 5 vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 @@ -998,3 +1016,13 @@ vec_is_equal_16x: csel x0, x0, xzr, eq ret .size vec_is_equal_16x,.-vec_is_equal_16x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_inverse_mod_256-armv8.S b/blst_src/build/elf/ct_inverse_mod_256-armv8.S index 7d4d2eb5..32e3d271 100644 --- a/blst_src/build/elf/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/elf/ct_inverse_mod_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_inverse_mod_256 @@ -5,7 +13,7 @@ .type ct_inverse_mod_256, %function .align 5 ct_inverse_mod_256: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -20,6 +28,7 @@ ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +48,7 @@ ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -48,18 +57,18 @@ ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +80,27 @@ ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +113,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +138,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +163,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +188,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +213,19 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +238,24 @@ ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +270,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +298,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +326,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +354,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +382,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +410,20 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +438,15 @@ ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +510,7 @@ ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_inverse_mod_256,.-ct_inverse_mod_256 @@ -579,11 +548,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +594,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain @@ -878,3 +847,13 @@ __inner_loop_62_256: ret .size __inner_loop_62_256,.-__inner_loop_62_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_inverse_mod_384-armv8.S b/blst_src/build/elf/ct_inverse_mod_384-armv8.S index daebd370..f507c54e 100644 --- a/blst_src/build/elf/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/elf/ct_inverse_mod_384-armv8.S @@ -1,11 +1,19 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383, %function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384, %function .align 5 -ct_inverse_mod_383: - .inst 0xd503233f +ct_inverse_mod_384: + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +30,7 @@ ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +54,40 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +105,269 @@ ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +377,7 @@ ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -375,20 +387,22 @@ ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +413,60 @@ ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,15 +480,15 @@ ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret -.size ct_inverse_mod_383,.-ct_inverse_mod_383 +.size ct_inverse_mod_384,.-ct_inverse_mod_384 //////////////////////////////////////////////////////////////////////// // see corresponding commentary in ctx_inverse_mod_384-x86_64... -.type __smul_383x63, %function +.type __smul_384x63, %function .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +496,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +511,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +552,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +582,41 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret -.size __smul_383x63,.-__smul_383x63 +.size __smul_384x63,.-__smul_384x63 -.type __smul_767x63_tail, %function +.type __smul_768x63_tail, %function .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +624,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +637,15 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret -.size __smul_767x63_tail,.-__smul_767x63_tail +.size __smul_768x63_tail,.-__smul_768x63_tail -.type __smul_383_n_shift_by_62, %function +.type __smul_384_n_shift_by_62, %function .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +665,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +705,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 @@ -700,7 +764,7 @@ __smul_383_n_shift_by_62: sub x16, x16, x28 ret -.size __smul_383_n_shift_by_62,.-__smul_383_n_shift_by_62 +.size __smul_384_n_shift_by_62,.-__smul_384_n_shift_by_62 .type __ab_approximation_62, %function .align 4 __ab_approximation_62: @@ -793,3 +857,13 @@ __inner_loop_62: ret .size __inner_loop_62,.-__inner_loop_62 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ct_is_square_mod_384-armv8.S b/blst_src/build/elf/ct_is_square_mod_384-armv8.S index ecac2453..950d49a0 100644 --- a/blst_src/build/elf/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/elf/ct_is_square_mod_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl ct_is_square_mod_384 @@ -5,7 +13,7 @@ .type ct_is_square_mod_384, %function .align 5 ct_is_square_mod_384: - .inst 0xd503233f + hint #PACI_HINT stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +92,7 @@ ct_is_square_mod_384: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size ct_is_square_mod_384,.-ct_is_square_mod_384 @@ -332,3 +340,13 @@ __inner_loop_48: ret .size __inner_loop_48,.-__inner_loop_48 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s b/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s index ce59e2c3..049532e4 100644 --- a/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/elf/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm __blst_platform_cap,4 .text -.globl ct_inverse_mod_383 -.hidden ct_inverse_mod_383 -.type ct_inverse_mod_383,@function +.globl ct_inverse_mod_384 +.hidden ct_inverse_mod_384 +.type ct_inverse_mod_384,@function .align 32 -ct_inverse_mod_383: +ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,__blst_platform_cap(%rip) - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -549,10 +589,10 @@ ct_inverse_mod_383: .byte 0xf3,0xc3 #endif .cfi_endproc -.size ct_inverse_mod_383,.-ct_inverse_mod_383 -.type __smulq_767x63,@function +.size ct_inverse_mod_384,.-ct_inverse_mod_384 +.type __smulq_768x63,@function .align 32 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -770,10 +817,10 @@ __smulq_767x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_767x63,.-__smulq_767x63 -.type __smulq_383x63,@function +.size __smulq_768x63,.-__smulq_768x63 +.type __smulq_384x63,@function .align 32 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -922,10 +984,10 @@ __smulq_383x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383x63,.-__smulq_383x63 -.type __smulq_383_n_shift_by_62,@function +.size __smulq_384x63,.-__smulq_384x63 +.type __smulq_384_n_shift_by_62,@function .align 32 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 @@ -1109,7 +1176,7 @@ __smulq_383_n_shift_by_62: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulq_383_n_shift_by_62,.-__smulq_383_n_shift_by_62 +.size __smulq_384_n_shift_by_62,.-__smulq_384_n_shift_by_62 .type __ab_approximation_62,@function .align 32 __ab_approximation_62: diff --git a/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s b/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s index a7bc5a5a..96e129d5 100644 --- a/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/elf/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl ctx_inverse_mod_383 -.hidden ctx_inverse_mod_383 -.type ctx_inverse_mod_383,@function +.globl ctx_inverse_mod_384 +.hidden ctx_inverse_mod_384 +.type ctx_inverse_mod_384,@function .align 32 -ctx_inverse_mod_383: +ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -ct_inverse_mod_383$1: +ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -893,10 +933,10 @@ ct_inverse_mod_383$1: .byte 0xf3,0xc3 #endif .cfi_endproc -.size ctx_inverse_mod_383,.-ctx_inverse_mod_383 -.type __smulx_767x63,@function +.size ctx_inverse_mod_384,.-ctx_inverse_mod_384 +.type __smulx_768x63,@function .align 32 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,62 +1046,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1068,10 +1115,10 @@ __smulx_767x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_767x63,.-__smulx_767x63 -.type __smulx_383x63,@function +.size __smulx_768x63,.-__smulx_768x63 +.type __smulx_384x63,@function .align 32 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,12 +1144,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1115,19 +1168,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,12 +1199,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1161,13 +1222,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1186,15 +1250,14 @@ __smulx_383x63: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383x63,.-__smulx_383x63 -.type __smulx_383_n_shift_by_31,@function +.size __smulx_384x63,.-__smulx_384x63 +.type __smulx_384_n_shift_by_31,@function .align 32 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1342,7 +1410,7 @@ __smulx_383_n_shift_by_31: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __smulx_383_n_shift_by_31,.-__smulx_383_n_shift_by_31 +.size __smulx_384_n_shift_by_31,.-__smulx_384_n_shift_by_31 .type __smulx_191_n_shift_by_31,@function .align 32 __smulx_191_n_shift_by_31: @@ -1585,9 +1653,9 @@ __inner_loop_31: .cfi_endproc .size __inner_loop_31,.-__inner_loop_31 -.type __tail_loop_53,@function +.type __tail_loop_55,@function .align 32 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -.Loop_53: +.Loop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ __tail_loop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz .Loop_53 + jnz .Loop_55 #ifdef __SGX_LVI_HARDENING__ @@ -1635,7 +1703,7 @@ __tail_loop_53: .byte 0xf3,0xc3 #endif .cfi_endproc -.size __tail_loop_53,.-__tail_loop_53 +.size __tail_loop_55,.-__tail_loop_55 .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/div3w-armv8.S b/blst_src/build/elf/div3w-armv8.S index 864773bf..48bbcb7d 100644 --- a/blst_src/build/elf/div3w-armv8.S +++ b/blst_src/build/elf/div3w-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl div_3_limbs @@ -5,6 +13,7 @@ .type div_3_limbs,%function .align 5 div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +48,7 @@ div_3_limbs: .type quot_rem_128,%function .align 5 quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +86,7 @@ quot_rem_128: .type quot_rem_64,%function .align 5 quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend @@ -89,3 +100,13 @@ quot_rem_64: ret .size quot_rem_64,.-quot_rem_64 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mul_mont_256-armv8.S b/blst_src/build/elf/mul_mont_256-armv8.S index b4ea3a6e..b8c39313 100644 --- a/blst_src/build/elf/mul_mont_256-armv8.S +++ b/blst_src/build/elf/mul_mont_256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl mul_mont_sparse_256 @@ -5,6 +13,7 @@ .type mul_mont_sparse_256,%function .align 5 mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +205,7 @@ mul_mont_sparse_256: .type sqr_mont_sparse_256,%function .align 5 sqr_mont_sparse_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +306,7 @@ sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_sparse_256,.-sqr_mont_sparse_256 .globl from_mont_256 @@ -305,7 +314,7 @@ sqr_mont_sparse_256: .type from_mont_256,%function .align 5 from_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +339,7 @@ from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_256,.-from_mont_256 @@ -339,7 +348,7 @@ from_mont_256: .type redc_mont_256,%function .align 5 redc_mont_256: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +383,7 @@ redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_256,.-redc_mont_256 @@ -462,3 +471,13 @@ __mul_by_1_mont_256: ret .size __mul_by_1_mont_256,.-__mul_by_1_mont_256 + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mul_mont_384-armv8.S b/blst_src/build/elf/mul_mont_384-armv8.S index ea120827..13f45234 100644 --- a/blst_src/build/elf/mul_mont_384-armv8.S +++ b/blst_src/build/elf/mul_mont_384-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + .text .globl add_mod_384x384 @@ -5,7 +13,7 @@ .type add_mod_384x384,%function .align 5 add_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -23,7 +31,7 @@ add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size add_mod_384x384,.-add_mod_384x384 @@ -87,7 +95,7 @@ __add_mod_384x384: .type sub_mod_384x384,%function .align 5 sub_mod_384x384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +113,7 @@ sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sub_mod_384x384,.-sub_mod_384x384 @@ -241,7 +249,7 @@ __sub_mod_384: .type mul_mont_384x,%function .align 5 mul_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -314,7 +322,7 @@ mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384x,.-mul_mont_384x @@ -323,7 +331,7 @@ mul_mont_384x: .type sqr_mont_384x,%function .align 5 sqr_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -399,7 +407,7 @@ sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384x,.-sqr_mont_384x @@ -408,7 +416,7 @@ sqr_mont_384x: .type mul_mont_384,%function .align 5 mul_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -440,7 +448,7 @@ mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_mont_384,.-mul_mont_384 @@ -821,7 +829,7 @@ __mul_mont_384: .type sqr_mont_384,%function .align 5 sqr_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -858,7 +866,7 @@ sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_mont_384,.-sqr_mont_384 @@ -867,7 +875,7 @@ sqr_mont_384: .type sqr_n_mul_mont_383,%function .align 5 sqr_n_mul_mont_383: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -923,7 +931,7 @@ sqr_n_mul_mont_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 .type __sqr_384,%function @@ -1044,7 +1052,7 @@ __sqr_384: .type sqr_384,%function .align 5 sqr_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1074,7 @@ sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_384,.-sqr_384 @@ -1075,7 +1083,7 @@ sqr_384: .type redc_mont_384,%function .align 5 redc_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1107,7 @@ redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size redc_mont_384,.-redc_mont_384 @@ -1108,7 +1116,7 @@ redc_mont_384: .type from_mont_384,%function .align 5 from_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1157,7 @@ from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size from_mont_384,.-from_mont_384 @@ -1367,7 +1375,7 @@ __redc_tail_mont_384: .type mul_384,%function .align 5 mul_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1393,7 @@ mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_384,.-mul_384 @@ -1571,7 +1579,7 @@ __mul_384: .type mul_382x,%function .align 5 mul_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1661,7 @@ mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size mul_382x,.-mul_382x @@ -1662,7 +1670,7 @@ mul_382x: .type sqr_382x,%function .align 5 sqr_382x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1753,480 +1761,16 @@ sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sqr_382x,.-sqr_382x -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,%function -.align 5 -sqr_mont_382x: - .inst 0xd503233f - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf - ret -.size sqr_mont_382x,.-sqr_mont_382x - -.type __mul_mont_383_nonred,%function -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - - ret -.size __mul_mont_383_nonred,.-__mul_mont_383_nonred - .globl sgn0_pty_mont_384 .hidden sgn0_pty_mont_384 .type sgn0_pty_mont_384,%function .align 5 sgn0_pty_mont_384: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1815,7 @@ sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384,.-sgn0_pty_mont_384 @@ -2280,7 +1824,7 @@ sgn0_pty_mont_384: .type sgn0_pty_mont_384x,%function .align 5 sgn0_pty_mont_384x: - .inst 0xd503233f + hint #PACI_HINT stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1913,16 @@ sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - .inst 0xd50323bf + hint #AUTI_HINT ret .size sgn0_pty_mont_384x,.-sgn0_pty_mont_384x + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/elf/mulq_mont_384-x86_64.s b/blst_src/build/elf/mulq_mont_384-x86_64.s index b5653128..377fe596 100644 --- a/blst_src/build/elf/mulq_mont_384-x86_64.s +++ b/blst_src/build/elf/mulq_mont_384-x86_64.s @@ -3101,768 +3101,6 @@ sqr_n_mul_mont_383: #endif .cfi_endproc .size sqr_n_mul_mont_383,.-sqr_n_mul_mont_383 -.type __mulq_mont_383_nonred,@function -.align 32 -__mulq_mont_383_nonred: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size __mulq_mont_383_nonred,.-__mulq_mont_383_nonred -.globl sqr_mont_382x -.hidden sqr_mont_382x -.type sqr_mont_382x,@function -.align 32 -sqr_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -#ifdef __BLST_PORTABLE__ - testl $1,__blst_platform_cap(%rip) - jnz sqr_mont_382x$1 -#endif - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size sqr_mont_382x,.-sqr_mont_382x .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/mulx_mont_256-x86_64.s b/blst_src/build/elf/mulx_mont_256-x86_64.s index 788c78df..989d68c5 100644 --- a/blst_src/build/elf/mulx_mont_256-x86_64.s +++ b/blst_src/build/elf/mulx_mont_256-x86_64.s @@ -202,17 +202,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -247,17 +244,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -292,16 +286,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -318,11 +309,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/elf/mulx_mont_384-x86_64.s b/blst_src/build/elf/mulx_mont_384-x86_64.s index 8d299432..8c4a6623 100644 --- a/blst_src/build/elf/mulx_mont_384-x86_64.s +++ b/blst_src/build/elf/mulx_mont_384-x86_64.s @@ -2028,8 +2028,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2057,11 +2057,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2091,8 +2089,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2120,11 +2118,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2154,8 +2150,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2183,11 +2179,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2217,8 +2211,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2246,11 +2240,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2280,8 +2272,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2309,11 +2301,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2344,10 +2334,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2735,9 +2724,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2795,9 +2783,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2855,9 +2842,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2915,9 +2901,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -2975,9 +2960,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3031,209 +3015,6 @@ __mulx_mont_383_nonred: #endif .cfi_endproc .size __mulx_mont_383_nonred,.-__mulx_mont_383_nonred -.globl sqrx_mont_382x -.hidden sqrx_mont_382x -.type sqrx_mont_382x,@function -.align 32 -sqrx_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -sqr_mont_382x$1: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc -.size sqrx_mont_382x,.-sqrx_mont_382x .section .note.GNU-stack,"",@progbits #ifndef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/elf/sha256-armv8.S b/blst_src/build/elf/sha256-armv8.S index 965f60eb..3766a3a7 100644 --- a/blst_src/build/elf/sha256-armv8.S +++ b/blst_src/build/elf/sha256-armv8.S @@ -1,3 +1,11 @@ +#if defined(__ARM_FEATURE_PAC_DEFAULT) && __ARM_FEATURE_PAC_DEFAULT==2 +# define PACI_HINT 27 +# define AUTI_HINT 31 +#else +# define PACI_HINT 25 +# define AUTI_HINT 29 +#endif + // // Copyright Supranational LLC // Licensed under the Apache License, Version 2.0, see LICENSE for details. @@ -47,6 +55,7 @@ .type blst_sha256_block_armv8,%function .align 6 blst_sha256_block_armv8: + hint #34 .Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -187,6 +196,7 @@ blst_sha256_block_armv8: .type blst_sha256_block_data_order,%function .align 4 blst_sha256_block_data_order: + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,#:lo12:__blst_platform_cap] tst w16,#1 @@ -1036,6 +1046,7 @@ blst_sha256_block_data_order: .type blst_sha256_emit,%function .align 4 blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1075,7 @@ blst_sha256_emit: .type blst_sha256_bcopy,%function .align 4 blst_sha256_bcopy: + hint #34 .Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1077,9 +1089,20 @@ blst_sha256_bcopy: .type blst_sha256_hcopy,%function .align 4 blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] stp x6,x7,[x0,#16] ret .size blst_sha256_hcopy,.-blst_sha256_hcopy + +#if defined(__ARM_FEATURE_BTI_DEFAULT) || defined(__ARM_FEATURE_PAC_DEFAULT) +.section .note.GNU-stack,"",@progbits +.section .note.gnu.property,"a",@note + .long 4,2f-1f,5 + .byte 0x47,0x4E,0x55,0 +1: .long 0xc0000000,4,3 +.align 3 +2: +#endif diff --git a/blst_src/build/mach-o/add_mod_256-armv8.S b/blst_src/build/mach-o/add_mod_256-armv8.S index 198d65ae..1a6e63f5 100644 --- a/blst_src/build/mach-o/add_mod_256-armv8.S +++ b/blst_src/build/mach-o/add_mod_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _add_mod_256: + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -39,6 +40,7 @@ _add_mod_256: .align 5 _mul_by_3_mod_256: + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -88,6 +90,7 @@ _mul_by_3_mod_256: .align 5 _lshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -126,6 +129,7 @@ Loop_lshift_mod_256: .align 5 _rshift_mod_256: + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/mach-o/add_mod_384-armv8.S b/blst_src/build/mach-o/add_mod_384-armv8.S index dedfa62f..c046b44a 100644 --- a/blst_src/build/mach-o/add_mod_384-armv8.S +++ b/blst_src/build/mach-o/add_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _add_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ _add_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -71,7 +71,7 @@ __add_mod_384_ab_are_loaded: .align 5 _add_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -99,7 +99,7 @@ _add_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -108,7 +108,7 @@ _add_mod_384x: .align 5 _rshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -135,7 +135,7 @@ Loop_rshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -170,7 +170,7 @@ __rshift_mod_384: .align 5 _div_by_2_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -194,7 +194,7 @@ _div_by_2_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -203,7 +203,7 @@ _div_by_2_mod_384: .align 5 _lshift_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -230,7 +230,7 @@ Loop_lshift_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -268,7 +268,7 @@ __lshift_mod_384: .align 5 _mul_by_3_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -298,7 +298,7 @@ _mul_by_3_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -307,7 +307,7 @@ _mul_by_3_mod_384: .align 5 _mul_by_8_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -333,7 +333,7 @@ _mul_by_8_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -342,7 +342,7 @@ _mul_by_8_mod_384: .align 5 _mul_by_3_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -387,7 +387,7 @@ _mul_by_3_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -396,7 +396,7 @@ _mul_by_3_mod_384x: .align 5 _mul_by_8_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -433,7 +433,7 @@ _mul_by_8_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -442,7 +442,7 @@ _mul_by_8_mod_384x: .align 5 _cneg_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -484,7 +484,7 @@ _cneg_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -493,7 +493,7 @@ _cneg_mod_384: .align 5 _sub_mod_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -513,7 +513,7 @@ _sub_mod_384: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -556,7 +556,7 @@ __sub_mod_384: .align 5 _sub_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -584,7 +584,7 @@ _sub_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -593,7 +593,7 @@ _sub_mod_384x: .align 5 _mul_by_1_plus_i_mod_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -626,7 +626,7 @@ _mul_by_1_plus_i_mod_384x: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -635,6 +635,7 @@ _mul_by_1_plus_i_mod_384x: .align 5 _sgn0_pty_mod_384: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -672,6 +673,7 @@ _sgn0_pty_mod_384: .align 5 _sgn0_pty_mod_384x: + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -753,14 +755,14 @@ _sgn0_pty_mod_384x: .align 5 _vec_select_32: + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret .globl _vec_select_48 @@ -768,6 +770,7 @@ _vec_select_32: .align 5 _vec_select_48: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -783,6 +786,7 @@ _vec_select_48: .align 5 _vec_select_96: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -804,6 +808,7 @@ _vec_select_96: .align 5 _vec_select_192: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -837,6 +842,7 @@ _vec_select_192: .align 5 _vec_select_144: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -864,6 +870,7 @@ _vec_select_144: .align 5 _vec_select_288: + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -909,6 +916,7 @@ _vec_select_288: .align 5 _vec_prefetch: + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -949,6 +957,7 @@ _vec_prefetch: .align 5 _vec_is_zero_16x: + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -974,6 +983,7 @@ Loop_is_zero_done: .align 5 _vec_is_equal_16x: + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S b/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S index 48cebab2..54a93e77 100644 --- a/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S +++ b/blst_src/build/mach-o/ct_inverse_mod_256-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_inverse_mod_256: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -20,6 +20,7 @@ _ct_inverse_mod_256: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #16+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -39,7 +40,7 @@ _ct_inverse_mod_256: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] // initialize |u| with |f0| @@ -48,18 +49,18 @@ _ct_inverse_mod_256: mov x13, x15 // |g1| add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] // initialize |v| with |f1| + str x12, [x0,#8*10] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -71,29 +72,27 @@ _ct_inverse_mod_256: bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] // |u| - ldr x9, [x1,#8*13] // |v| + ldr x9, [x1,#8*14] // |v| madd x4, x16, x8, xzr // |u|*|f0| madd x4, x17, x9, x4 // |v|*|g0| - str x4, [x0,#8*4] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr // |u|*|f1| madd x4, x13, x9, x4 // |v|*|g1| - str x4, [x0,#8*9] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -106,25 +105,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -137,25 +130,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -168,25 +155,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -199,25 +180,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -230,25 +205,19 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -261,25 +230,24 @@ _ct_inverse_mod_256: add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 // sign extension + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -294,21 +262,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -323,21 +290,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -352,21 +318,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -381,21 +346,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -410,21 +374,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -439,21 +402,20 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif bl __ab_approximation_31_256 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif bl __smul_256_n_shift_by_31 mov x16, x12 // corrected |f0| @@ -468,16 +430,15 @@ _ct_inverse_mod_256: bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 // corrected |f1| mov x17, x13 // corrected |g1| - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail ////////////////////////////////////////// two[!] last iterations eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #47 // 31 + 512 % 31 //bl __ab_approximation_62_256 // |a| and |b| are exact, @@ -541,7 +502,7 @@ _ct_inverse_mod_256: ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -579,11 +540,11 @@ __smul_256x63: adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] // load |u| (or |v|) + ldp x8, x9, [x1,#8*0+112] // load |u| (or |v|) asr x14, x17, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 // conditionally negate |f_| (or |g_|) - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 // conditionally negate |u| (or |v|) sub x17, x17, x14 @@ -625,9 +586,9 @@ __smul_256x63: .align 5 __smul_512x63_tail: umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] // load rest of |v| + ldr x5, [x1,#8*19] // load rest of |v| adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 // resume |v|*|g1| chain diff --git a/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S b/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S index f1bb996f..07bdb673 100644 --- a/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S +++ b/blst_src/build/mach-o/ct_inverse_mod_384-armv8.S @@ -1,11 +1,11 @@ .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .align 5 -_ct_inverse_mod_383: -.long 3573752639 +_ct_inverse_mod_384: + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ _ct_inverse_mod_383: #ifdef __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 #else add x1, sp, #32+511 // find closest 512-byte-aligned spot and x1, x1, #-512 // in the frame... @@ -45,40 +46,40 @@ _ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] // initialize |u| with |f0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] // initialize |v| with |f1| + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] // initialize |v| with |f1| ////////////////////////////////////////// second iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] // |u| - ldr x8, [x1,#8*18] // |v| + ldr x8, [x1,#8*20] // |v| mul x3, x20, x7 // |u|*|f0| smulh x4, x20, x7 mul x5, x21, x8 // |v|*|g0| @@ -96,266 +97,269 @@ _ct_inverse_mod_383: smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 // sign extension - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 // sign extension stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 // corrected |f0| mov x21, x16 // corrected |g0| mov x15, x17 // |f1| mov x16, x19 // |g1| add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // corrected |f1| mov x21, x16 // corrected |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// iteration before last eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif mov x2, #62 //bl __ab_approximation_62 // |a| and |b| are exact, @@ -365,7 +369,7 @@ _ct_inverse_mod_383: eor x0, x1, #256 // pointer to dst |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 #endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -375,20 +379,22 @@ _ct_inverse_mod_383: mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 // exact |f1| mov x21, x16 // exact |g1| - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail ////////////////////////////////////////// last iteration eor x1, x1, #256 // flip-flop src |a|b|u|v| #ifdef __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 #endif - mov x2, #22 // 766 % 62 + mov x2, #24 // 768 % 62 //bl __ab_approximation_62 // |a| and |b| are exact, ldr x3, [x1,#8*0] // just load eor x8, x8, x8 @@ -399,25 +405,60 @@ _ct_inverse_mod_383: mov x20, x17 mov x21, x19 ldp x0, x15, [sp] // original out_ptr and n_ptr - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 // sign as mask - ldp x9, x10, [x15,#8*0] + smulh x23, x8, x21 // figure out top-most limb + adc x26, x26, x28 + ldp x9, x10, [x15,#8*0] // load |mod| + add x23, x23, x26 // x23 is 1, 0 or -1 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 // sign as mask ldp x13, x14, [x15,#8*4] - and x9, x9, x22 // add mod<<384 conditionally - and x10, x10, x22 - adds x3, x3, x9 - and x11, x11, x22 + and x26, x9, x22 // add mod<<384 conditionally + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr // x23 is 1, 0 or -1 + + neg x22, x23 + orr x23, x23, x22 // excess bit or sign as mask + asr x22, x22, #63 // excess bit as mask + + and x9, x9, x23 // mask |mod| + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 // conditionally negate |mod| + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + + adds x3, x3, x9 // final adjustment for |mod|<<384 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -431,7 +472,7 @@ _ct_inverse_mod_383: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -439,7 +480,7 @@ _ct_inverse_mod_383: // see corresponding commentary in ctx_inverse_mod_384-x86_64... .align 5 -__smul_383x63: +__smul_384x63: ldp x3, x4, [x1,#8*0+96] // load |u| (or |v|) asr x17, x20, #63 // |f_|'s sign as mask (or |g_|'s) ldp x5, x6, [x1,#8*2+96] @@ -447,6 +488,7 @@ __smul_383x63: ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 // conditionally negate |u| (or |v|) + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -461,28 +503,33 @@ __smul_383x63: umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + csel x25, x25, xzr, ne adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] // load |u| (or |v|) + ldp x9, x10, [x1,#8*0+160] // load |u| (or |v|) asr x17, x21, #63 // |f_|'s sign as mask (or |g_|'s) - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 // conditionally negate |f_| (or |g_|) - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 // conditionally negate |u| (or |v|) + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -497,21 +544,25 @@ __smul_383x63: umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr // used in __smul_767x63_tail - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr // used in __smul_768x63_tail + cmp x21, #0 mul x11, x11, x21 + csel x26, x26, xzr, ne adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -523,41 +574,41 @@ __smul_383x63: stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr // used in __smul_767x63_tail ret .align 5 -__smul_767x63_tail: - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] // load rest of |v| - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] - - eor x3, x3, x17 // conditionally negate rest of |v| - eor x4, x4, x17 +__smul_768x63_tail: + umulh x27, x8, x20 + ldr x4, [x1,#8*27]// load rest of |v| + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 // tie up |u|*|f1| chain + + umulh x14, x14, x21 // resume |v|*|g1| chain + eor x4, x4, x17 // conditionally negate rest of |v| eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -565,10 +616,11 @@ __smul_767x63_tail: adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr // used in the final step adds x3, x3, x27 adcs x4, x4, x28 @@ -577,15 +629,15 @@ __smul_767x63_tail: stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 // carry is used in the final step + stp x7, x25, [x0,#8*10] ret .align 5 -__smul_383_n_shift_by_62: +__smul_384_n_shift_by_62: ldp x3, x4, [x1,#8*0+0] // load |a| (or |b|) asr x28, x15, #63 // |f0|'s sign as mask (or |g0|'s) ldp x5, x6, [x1,#8*2+0] @@ -605,25 +657,27 @@ __smul_383_n_shift_by_62: adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] // load |a| (or |b|) asr x28, x16, #63 // |f0|'s sign as mask (or |g0|'s) ldp x11, x12, [x1,#8*2+48] @@ -643,25 +697,27 @@ __smul_383_n_shift_by_62: adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S b/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S index fe9224af..1e61ea46 100644 --- a/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S +++ b/blst_src/build/mach-o/ct_is_square_mod_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _ct_is_square_mod_384: -.long 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -84,7 +84,7 @@ Loop_is_square: ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s b/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s index 0a37c731..c977ae13 100644 --- a/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s +++ b/blst_src/build/mach-o/ctq_inverse_mod_384-x86_64.s @@ -1,18 +1,18 @@ .comm ___blst_platform_cap,4 .text -.globl _ct_inverse_mod_383 -.private_extern _ct_inverse_mod_383 +.globl _ct_inverse_mod_384 +.private_extern _ct_inverse_mod_384 .p2align 5 -_ct_inverse_mod_383: +_ct_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa #ifdef __BLST_PORTABLE__ testl $1,___blst_platform_cap(%rip) - jnz L$ct_inverse_mod_383$1 + jnz L$ct_inverse_mod_384$1 #endif pushq %rbp .cfi_adjust_cfa_offset 8 @@ -80,7 +80,7 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,96(%rdi) @@ -88,10 +88,10 @@ _ct_inverse_mod_383: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -104,19 +104,19 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -133,6 +133,7 @@ _ct_inverse_mod_383: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -143,13 +144,14 @@ _ct_inverse_mod_383: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -160,14 +162,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -175,12 +177,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -191,14 +193,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -206,12 +208,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -222,14 +224,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -237,12 +239,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 + leaq 56(%rdi),%rdi + call __smulq_384x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -253,14 +255,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -268,19 +270,17 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulq_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -291,14 +291,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -306,12 +306,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -322,14 +322,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -337,12 +337,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -353,14 +353,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -368,12 +368,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -384,14 +384,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -399,12 +399,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi call __ab_approximation_62 @@ -415,14 +415,14 @@ _ct_inverse_mod_383: movq $256,%rdi xorq %rsi,%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -430,12 +430,12 @@ _ct_inverse_mod_383: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi movl $62,%edi @@ -459,16 +459,16 @@ _ct_inverse_mod_383: leaq 96(%rsi),%rsi leaq 96(%rdi),%rdi - call __smulq_383x63 + call __smulq_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulq_767x63 + leaq 56(%rdi),%rdi + call __smulq_768x63 xorq $256+96,%rsi - movl $22,%edi + movl $24,%edi movq 0(%rsi),%r8 xorq %r9,%r9 @@ -491,37 +491,77 @@ _ct_inverse_mod_383: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulq_767x63 + call __smulq_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -552,7 +592,7 @@ _ct_inverse_mod_383: .p2align 5 -__smulq_767x63: +__smulq_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -562,6 +602,7 @@ __smulq_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -570,7 +611,7 @@ __smulq_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rdx,%rbp addq %rax,%rbp @@ -581,16 +622,20 @@ __smulq_767x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,0(%rdi) movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -616,14 +661,14 @@ __smulq_767x63: adcq $0,%rdx movq %rdx,%r13 movq %r12,32(%rdi) - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 movq %r13,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq 0(%rsi),%r8 @@ -726,39 +771,41 @@ __smulq_767x63: movq %rdi,%rax adcq $0,%rdx movq %rdx,%rdi - movq 8(%rsp),%rdx - imulq %rsi,%rax - movq 16(%rsp),%rsi + imulq %rsi + movq 8(%rsp),%rsi addq %rdi,%rax + adcq $0,%rdx - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi + addq 0(%rsi),%r8 + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + adcq 24(%rsi),%r11 + adcq 32(%rsi),%r12 + adcq 40(%rsi),%r13 + adcq 48(%rsi),%r14 + movq 56(%rsi),%rdi adcq %rdi,%r15 adcq %rdi,%rbx adcq %rdi,%rbp adcq %rdi,%rcx adcq %rdi,%rax + adcq %rdi,%rdx - movq %rdx,%rdi + leaq (%rsi),%rdi + movq 16(%rsp),%rsi - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -773,7 +820,7 @@ __smulq_767x63: .p2align 5 -__smulq_383x63: +__smulq_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -783,6 +830,7 @@ __smulq_383x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -798,16 +846,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -829,10 +881,11 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi movq %rcx,%rdx movq %r8,0(%rdi) @@ -840,13 +893,15 @@ __smulq_383x63: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) + movq %r13,%r15 + movq %r14,%rbx movq 0(%rsi),%r8 movq 8(%rsi),%r9 movq 16(%rsi),%r10 movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rdx @@ -862,16 +917,20 @@ __smulq_383x63: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + xorq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -893,17 +952,19 @@ __smulq_383x63: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp,%rax + mulq %rbp addq %rax,%r13 + adcq %rdx,%r14 - leaq -48(%rsi),%rsi + leaq -56(%rsi),%rsi addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -911,6 +972,7 @@ __smulq_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -925,7 +987,7 @@ __smulq_383x63: .p2align 5 -__smulq_383_n_shift_by_62: +__smulq_384_n_shift_by_62: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -951,6 +1013,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r14 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -961,6 +1024,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r14 + negq %r14 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -982,12 +1047,11 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r14 leaq 48(%rsi),%rsi - movq %rdx,%r14 movq %rcx,%rdx movq %r8,0(%rdi) @@ -1017,6 +1081,7 @@ __smulq_383_n_shift_by_62: xorq %rdx,%r11 xorq %rdx,%r12 xorq %rdx,%r13 + movq %rdx,%r15 addq %r8,%rax adcq $0,%r9 adcq $0,%r10 @@ -1027,6 +1092,8 @@ __smulq_383_n_shift_by_62: mulq %rbp movq %rax,%r8 movq %r9,%rax + andq %rbp,%r15 + negq %r15 movq %rdx,%r9 mulq %rbp addq %rax,%r9 @@ -1048,11 +1115,12 @@ __smulq_383_n_shift_by_62: movq %r13,%rax adcq $0,%rdx movq %rdx,%r13 - imulq %rbp + mulq %rbp addq %rax,%r13 - adcq $0,%rdx + adcq %rdx,%r15 leaq -48(%rsi),%rsi + movq %rbx,%rdx addq 0(%rdi),%r8 adcq 8(%rdi),%r9 @@ -1060,8 +1128,7 @@ __smulq_383_n_shift_by_62: adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 adcq 40(%rdi),%r13 - adcq %rdx,%r14 - movq %rbx,%rdx + adcq %r15,%r14 shrdq $62,%r9,%r8 shrdq $62,%r10,%r9 diff --git a/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s b/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s index e494e44b..aad4e1a4 100644 --- a/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s +++ b/blst_src/build/mach-o/ctx_inverse_mod_384-x86_64.s @@ -1,15 +1,15 @@ .text -.globl _ctx_inverse_mod_383 -.private_extern _ctx_inverse_mod_383 +.globl _ctx_inverse_mod_384 +.private_extern _ctx_inverse_mod_384 .p2align 5 -_ctx_inverse_mod_383: +_ctx_inverse_mod_384: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa -L$ct_inverse_mod_383$1: +L$ct_inverse_mod_384$1: pushq %rbp .cfi_adjust_cfa_offset 8 .cfi_offset %rbp,-16 @@ -79,7 +79,7 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,96(%rdi) @@ -87,10 +87,10 @@ L$ct_inverse_mod_383$1: movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - movq %rdx,96(%rdi) + movq %rdx,104(%rdi) xorq $256,%rsi @@ -103,19 +103,19 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq 96(%rsi),%rax - movq 144(%rsi),%r11 + movq 152(%rsi),%r11 movq %rdx,%rbx movq %rax,%r10 imulq 56(%rsp) @@ -132,6 +132,7 @@ L$ct_inverse_mod_383$1: movq %r9,72(%rdi) movq %r9,80(%rdi) movq %r9,88(%rdi) + movq %r9,96(%rdi) leaq 96(%rsi),%rsi movq %r10,%rax @@ -142,13 +143,14 @@ L$ct_inverse_mod_383$1: imulq %rcx addq %rax,%r8 adcq %rdx,%r9 - movq %r8,96(%rdi) - movq %r9,104(%rdi) - sarq $63,%r9 + movq %r8,104(%rdi) movq %r9,112(%rdi) + sarq $63,%r9 movq %r9,120(%rdi) movq %r9,128(%rdi) movq %r9,136(%rdi) + movq %r9,144(%rdi) + movq %r9,152(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -159,14 +161,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -174,12 +176,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -190,14 +192,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -205,12 +207,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -221,14 +223,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -236,12 +238,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -252,14 +254,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -267,12 +269,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -283,14 +285,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -298,12 +300,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -314,14 +316,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -329,12 +331,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -345,14 +347,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -360,12 +362,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -376,14 +378,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -391,12 +393,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -407,14 +409,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -422,12 +424,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 + leaq 56(%rdi),%rdi + call __smulx_384x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -438,14 +440,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -453,19 +455,17 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_383x63 - sarq $63,%r13 - movq %r13,48(%rdi) - movq %r13,56(%rdi) - movq %r13,64(%rdi) - movq %r13,72(%rdi) - movq %r13,80(%rdi) - movq %r13,88(%rdi) + leaq 56(%rdi),%rdi + call __smulx_384x63 + movq %r14,56(%rdi) + movq %r14,64(%rdi) + movq %r14,72(%rdi) + movq %r14,80(%rdi) + movq %r14,88(%rdi) xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -476,14 +476,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -491,12 +491,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -507,14 +507,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -522,12 +522,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -538,14 +538,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -553,12 +553,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -569,14 +569,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -584,12 +584,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -600,14 +600,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -615,12 +615,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -631,14 +631,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -646,12 +646,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -662,14 +662,14 @@ L$ct_inverse_mod_383$1: movq $256,%rdi xorq %rsi,%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,56(%rsp) movq %rcx,64(%rsp) movq 72(%rsp),%rdx movq 80(%rsp),%rcx leaq 48(%rdi),%rdi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 movq %rdx,72(%rsp) movq %rcx,80(%rsp) @@ -677,12 +677,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -708,12 +708,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -739,12 +739,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -770,12 +770,12 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi movl $31,%edi call __ab_approximation_31 @@ -801,21 +801,21 @@ L$ct_inverse_mod_383$1: movq 64(%rsp),%rcx leaq 96(%rsi),%rsi leaq 48(%rdi),%rdi - call __smulx_383x63 + call __smulx_384x63 movq 72(%rsp),%rdx movq 80(%rsp),%rcx - leaq 48(%rdi),%rdi - call __smulx_767x63 + leaq 56(%rdi),%rdi + call __smulx_768x63 xorq $256+96,%rsi - movl $53,%edi + movl $55,%edi movq 0(%rsi),%r8 movq 48(%rsi),%r10 - call __tail_loop_53 + call __tail_loop_55 @@ -832,40 +832,80 @@ L$ct_inverse_mod_383$1: movq %r12,%rdx movq %r13,%rcx movq 32(%rsp),%rdi - call __smulx_767x63 + call __smulx_768x63 movq 40(%rsp),%rsi - movq %rax,%rdx - sarq $63,%rax + movq %rdx,%r13 + sarq $63,%r13 - movq %rax,%r8 - movq %rax,%r9 - movq %rax,%r10 + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 #ifdef __SGX_LVI_HARDENING__ lfence #endif andq 0(%rsi),%r8 andq 8(%rsi),%r9 - movq %rax,%r11 + movq %r13,%r11 andq 16(%rsi),%r10 andq 24(%rsi),%r11 - movq %rax,%r12 + movq %r13,%r12 andq 32(%rsi),%r12 - andq 40(%rsi),%rax + andq 40(%rsi),%r13 addq %r8,%r14 adcq %r9,%r15 adcq %r10,%rbx adcq %r11,%rbp adcq %r12,%rcx - adcq %rax,%rdx + adcq %r13,%rax + adcq $0,%rdx + + movq %rdx,%r13 + negq %rdx + orq %rdx,%r13 + sarq $63,%rdx + + movq %r13,%r8 + movq %r13,%r9 + movq %r13,%r10 + andq 0(%rsi),%r8 + andq 8(%rsi),%r9 + movq %r13,%r11 + andq 16(%rsi),%r10 + andq 24(%rsi),%r11 + movq %r13,%r12 + andq 32(%rsi),%r12 + andq 40(%rsi),%r13 + + xorq %rdx,%r8 + xorq %rsi,%rsi + xorq %rdx,%r9 + subq %rdx,%rsi + xorq %rdx,%r10 + xorq %rdx,%r11 + xorq %rdx,%r12 + xorq %rdx,%r13 + addq %rsi,%r8 + adcq $0,%r9 + adcq $0,%r10 + adcq $0,%r11 + adcq $0,%r12 + adcq $0,%r13 + + addq %r8,%r14 + adcq %r9,%r15 + adcq %r10,%rbx + adcq %r11,%rbp + adcq %r12,%rcx + adcq %r13,%rax movq %r14,48(%rdi) movq %r15,56(%rdi) movq %rbx,64(%rdi) movq %rbp,72(%rdi) movq %rcx,80(%rdi) - movq %rdx,88(%rdi) + movq %rax,88(%rdi) leaq 1112(%rsp),%r8 movq 0(%r8),%r15 @@ -896,7 +936,7 @@ L$ct_inverse_mod_383$1: .p2align 5 -__smulx_767x63: +__smulx_768x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -906,6 +946,7 @@ __smulx_767x63: movq 24(%rsi),%r11 movq 32(%rsi),%r12 movq 40(%rsi),%r13 + movq 48(%rsi),%r14 movq %rdx,%rax sarq $63,%rax @@ -914,7 +955,7 @@ __smulx_767x63: movq %rdi,8(%rsp) movq %rsi,16(%rsp) - leaq 48(%rsi),%rsi + leaq 56(%rsi),%rsi xorq %rax,%rdx addq %rbp,%rdx @@ -924,37 +965,41 @@ __smulx_767x63: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 + xorq %rax,%r14 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%rax addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %rax,%r10 + mulxq %r11,%r11,%rax adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %rax,%r12 + mulxq %r13,%r13,%rax + adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) - movq %rdx,48(%rdi) - sarq $63,%rdx - movq %rdx,56(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + sarq $63,%r14 + movq %r14,56(%rdi) movq %rcx,%rdx movq %rcx,%rax @@ -989,7 +1034,7 @@ __smulx_767x63: xorq %rax,%rbx xorq %rax,%rbp xorq %rax,%rcx - xorq %rax,%rdi + xorq %rdi,%rax addq %rsi,%r8 adcq $0,%r9 adcq $0,%r10 @@ -1001,62 +1046,64 @@ __smulx_767x63: adcq $0,%rbx adcq $0,%rbp adcq $0,%rcx - adcq $0,%rdi - - mulxq %r8,%r8,%rax - mulxq %r9,%r9,%rsi - addq %rax,%r9 - mulxq %r10,%r10,%rax - adcq %rsi,%r10 - mulxq %r11,%r11,%rsi - adcq %rax,%r11 - mulxq %r12,%r12,%rax - adcq %rsi,%r12 - mulxq %r13,%r13,%rsi - adcq %rax,%r13 - mulxq %r14,%r14,%rax - adcq %rsi,%r14 - mulxq %r15,%r15,%rsi - adcq %rax,%r15 - mulxq %rbx,%rbx,%rax + adcq $0,%rax + + mulxq %r8,%r8,%rsi + mulxq %r9,%r9,%rdi + addq %rsi,%r9 + mulxq %r10,%r10,%rsi + adcq %rdi,%r10 + mulxq %r11,%r11,%rdi + adcq %rsi,%r11 + mulxq %r12,%r12,%rsi + adcq %rdi,%r12 + mulxq %r13,%r13,%rdi + adcq %rsi,%r13 + mulxq %r14,%r14,%rsi + adcq %rdi,%r14 + mulxq %r15,%r15,%rdi + adcq %rsi,%r15 + mulxq %rbx,%rbx,%rsi + adcq %rdi,%rbx + mulxq %rbp,%rbp,%rdi + adcq %rsi,%rbp + mulxq %rcx,%rcx,%rsi + adcq %rdi,%rcx + movq 8(%rsp),%rdi + adcq $0,%rsi + imulq %rdx + addq %rsi,%rax + adcq $0,%rdx + + addq 0(%rdi),%r8 + adcq 8(%rdi),%r9 + adcq 16(%rdi),%r10 + adcq 24(%rdi),%r11 + adcq 32(%rdi),%r12 + adcq 40(%rdi),%r13 + adcq 48(%rdi),%r14 + movq 56(%rdi),%rsi + adcq %rsi,%r15 adcq %rsi,%rbx - mulxq %rbp,%rbp,%rsi - adcq %rax,%rbp - mulxq %rcx,%rcx,%rax + adcq %rsi,%rbp adcq %rsi,%rcx - mulxq %rdi,%rdi,%rsi - movq 8(%rsp),%rdx + adcq %rsi,%rax + adcq %rsi,%rdx + movq 16(%rsp),%rsi - adcq %rdi,%rax - - addq 0(%rdx),%r8 - adcq 8(%rdx),%r9 - adcq 16(%rdx),%r10 - adcq 24(%rdx),%r11 - adcq 32(%rdx),%r12 - adcq 40(%rdx),%r13 - adcq 48(%rdx),%r14 - movq 56(%rdx),%rdi - adcq %rdi,%r15 - adcq %rdi,%rbx - adcq %rdi,%rbp - adcq %rdi,%rcx - adcq %rdi,%rax - - movq %rdx,%rdi - - movq %r8,0(%rdx) - movq %r9,8(%rdx) - movq %r10,16(%rdx) - movq %r11,24(%rdx) - movq %r12,32(%rdx) - movq %r13,40(%rdx) - movq %r14,48(%rdx) - movq %r15,56(%rdx) - movq %rbx,64(%rdx) - movq %rbp,72(%rdx) - movq %rcx,80(%rdx) - movq %rax,88(%rdx) + + movq %r8,0(%rdi) + movq %r9,8(%rdi) + movq %r10,16(%rdi) + movq %r11,24(%rdi) + movq %r12,32(%rdi) + movq %r13,40(%rdi) + movq %r14,48(%rdi) + movq %r15,56(%rdi) + movq %rbx,64(%rdi) + movq %rbp,72(%rdi) + movq %rcx,80(%rdi) + movq %rax,88(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1071,7 +1118,7 @@ __smulx_767x63: .p2align 5 -__smulx_383x63: +__smulx_384x63: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1081,6 +1128,7 @@ __smulx_383x63: movq 0+24(%rsi),%r11 movq 0+32(%rsi),%r12 movq 0+40(%rsi),%r13 + movq 0+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1096,12 +1144,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1115,19 +1168,22 @@ __smulx_383x63: mulxq %r13,%r13,%rax movq %rcx,%rdx adcq %rbp,%r13 + adcq %rax,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %r13,40(%rdi) - movq 48+0(%rsi),%r8 - movq 48+8(%rsi),%r9 - movq 48+16(%rsi),%r10 - movq 48+24(%rsi),%r11 - movq 48+32(%rsi),%r12 - movq 48+40(%rsi),%r13 + movq %r13,%r15 + movq %r14,%rbx + movq 56+0(%rsi),%r8 + movq 56+8(%rsi),%r9 + movq 56+16(%rsi),%r10 + movq 56+24(%rsi),%r11 + movq 56+32(%rsi),%r12 + movq 56+40(%rsi),%r13 + movq 56+48(%rsi),%r14 movq %rdx,%rbp sarq $63,%rbp @@ -1143,12 +1199,17 @@ __smulx_383x63: xorq %rbp,%r11 xorq %rbp,%r12 xorq %rbp,%r13 + xorq %rbp,%r14 addq %rax,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 adcq $0,%r13 + adcq $0,%r14 + + andq %rdx,%r14 + negq %r14 mulxq %r8,%r8,%rbp mulxq %r9,%r9,%rax @@ -1161,13 +1222,15 @@ __smulx_383x63: adcq %rax,%r12 mulxq %r13,%r13,%rax adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%r13 + adcq %r15,%r13 + adcq %rbx,%r14 movq %r8,0(%rdi) movq %r9,8(%rdi) @@ -1175,6 +1238,7 @@ __smulx_383x63: movq %r11,24(%rdi) movq %r12,32(%rdi) movq %r13,40(%rdi) + movq %r14,48(%rdi) #ifdef __SGX_LVI_HARDENING__ @@ -1189,12 +1253,11 @@ __smulx_383x63: .p2align 5 -__smulx_383_n_shift_by_31: +__smulx_384_n_shift_by_31: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa movq %rdx,%rbx - xorq %r14,%r14 movq 0+0(%rsi),%r8 movq 0+8(%rsi),%r9 movq 0+16(%rsi),%r10 @@ -1215,27 +1278,29 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq %rdx,%r14 + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 movq %rcx,%rdx @@ -1244,7 +1309,8 @@ __smulx_383_n_shift_by_31: movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) + movq %r14,%r15 movq 48+0(%rsi),%r8 movq 48+8(%rsi),%r9 movq 48+16(%rsi),%r10 @@ -1265,43 +1331,45 @@ __smulx_383_n_shift_by_31: xorq %rax,%r10 xorq %rax,%r11 xorq %rax,%r12 - xorq %r13,%rax + xorq %rax,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 + + andq %rdx,%rax + negq %rax mulxq %r8,%r8,%rbp - mulxq %r9,%r9,%r13 + mulxq %r9,%r9,%r14 addq %rbp,%r9 mulxq %r10,%r10,%rbp - adcq %r13,%r10 - mulxq %r11,%r11,%r13 + adcq %r14,%r10 + mulxq %r11,%r11,%r14 adcq %rbp,%r11 mulxq %r12,%r12,%rbp - adcq %r13,%r12 - adcq $0,%rbp - imulq %rdx - addq %rbp,%rax - adcq $0,%rdx + adcq %r14,%r12 + mulxq %r13,%r13,%r14 + adcq %rbp,%r13 + adcq %rax,%r14 addq 0(%rdi),%r8 adcq 8(%rdi),%r9 adcq 16(%rdi),%r10 adcq 24(%rdi),%r11 adcq 32(%rdi),%r12 - adcq 40(%rdi),%rax - adcq %rdx,%r14 + adcq 40(%rdi),%r13 + adcq %r15,%r14 movq %rbx,%rdx shrdq $31,%r9,%r8 shrdq $31,%r10,%r9 shrdq $31,%r11,%r10 shrdq $31,%r12,%r11 - shrdq $31,%rax,%r12 - shrdq $31,%r14,%rax + shrdq $31,%r13,%r12 + shrdq $31,%r14,%r13 sarq $63,%r14 xorq %rbp,%rbp @@ -1312,20 +1380,20 @@ __smulx_383_n_shift_by_31: xorq %r14,%r10 xorq %r14,%r11 xorq %r14,%r12 - xorq %r14,%rax + xorq %r14,%r13 addq %rbp,%r8 adcq $0,%r9 adcq $0,%r10 adcq $0,%r11 adcq $0,%r12 - adcq $0,%rax + adcq $0,%r13 movq %r8,0(%rdi) movq %r9,8(%rdi) movq %r10,16(%rdi) movq %r11,24(%rdi) movq %r12,32(%rdi) - movq %rax,40(%rdi) + movq %r13,40(%rdi) xorq %r14,%rdx xorq %r14,%rcx @@ -1587,7 +1655,7 @@ L$oop_31: .p2align 5 -__tail_loop_53: +__tail_loop_55: .cfi_startproc .byte 0xf3,0x0f,0x1e,0xfa @@ -1596,7 +1664,7 @@ __tail_loop_53: xorq %r12,%r12 movq $1,%r13 -L$oop_53: +L$oop_55: xorq %rax,%rax testq $1,%r8 movq %r10,%rbx @@ -1623,7 +1691,7 @@ L$oop_53: subq %rax,%rdx subq %rbx,%rcx subl $1,%edi - jnz L$oop_53 + jnz L$oop_55 #ifdef __SGX_LVI_HARDENING__ diff --git a/blst_src/build/mach-o/div3w-armv8.S b/blst_src/build/mach-o/div3w-armv8.S index 55bec60a..6530fb8f 100644 --- a/blst_src/build/mach-o/div3w-armv8.S +++ b/blst_src/build/mach-o/div3w-armv8.S @@ -5,6 +5,7 @@ .align 5 _div_3_limbs: + hint #34 ldp x4,x5,[x0] // load R eor x0,x0,x0 // Q = 0 mov x3,#64 // loop counter @@ -39,6 +40,7 @@ Loop: .align 5 _quot_rem_128: + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 // divisor[0:1} * quotient @@ -76,6 +78,7 @@ _quot_rem_128: .align 5 _quot_rem_64: + hint #34 ldr x3,[x1] ldr x8,[x0] // load 1 limb of the dividend diff --git a/blst_src/build/mach-o/mul_mont_256-armv8.S b/blst_src/build/mach-o/mul_mont_256-armv8.S index 3fac56fd..e1b9d454 100644 --- a/blst_src/build/mach-o/mul_mont_256-armv8.S +++ b/blst_src/build/mach-o/mul_mont_256-armv8.S @@ -5,6 +5,7 @@ .align 5 _mul_mont_sparse_256: + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +197,7 @@ _mul_mont_sparse_256: .align 5 _sqr_mont_sparse_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -297,7 +298,7 @@ _sqr_mont_sparse_256: ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret .globl _from_mont_256 @@ -305,7 +306,7 @@ _sqr_mont_sparse_256: .align 5 _from_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -330,7 +331,7 @@ _from_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -339,7 +340,7 @@ _from_mont_256: .align 5 _redc_mont_256: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -374,7 +375,7 @@ _redc_mont_256: stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/mul_mont_384-armv8.S b/blst_src/build/mach-o/mul_mont_384-armv8.S index 3c069ed7..77ca24d7 100644 --- a/blst_src/build/mach-o/mul_mont_384-armv8.S +++ b/blst_src/build/mach-o/mul_mont_384-armv8.S @@ -5,7 +5,7 @@ .align 5 _add_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -23,7 +23,7 @@ _add_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -87,7 +87,7 @@ __add_mod_384x384: .align 5 _sub_mod_384x384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -105,7 +105,7 @@ _sub_mod_384x384: ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -241,7 +241,7 @@ __sub_mod_384: .align 5 _mul_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -314,7 +314,7 @@ _mul_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -323,7 +323,7 @@ _mul_mont_384x: .align 5 _sqr_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -399,7 +399,7 @@ _sqr_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -408,7 +408,7 @@ _sqr_mont_384x: .align 5 _mul_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -440,7 +440,7 @@ _mul_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -821,7 +821,7 @@ __mul_mont_384: .align 5 _sqr_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -858,7 +858,7 @@ _sqr_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -867,7 +867,7 @@ _sqr_mont_384: .align 5 _sqr_n_mul_mont_383: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -923,7 +923,7 @@ Loop_sqr_383: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1044,7 +1044,7 @@ __sqr_384: .align 5 _sqr_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1066,7 +1066,7 @@ _sqr_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1075,7 +1075,7 @@ _sqr_384: .align 5 _redc_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1099,7 +1099,7 @@ _redc_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1108,7 +1108,7 @@ _redc_mont_384: .align 5 _from_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1149,7 +1149,7 @@ _from_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1367,7 +1367,7 @@ __redc_tail_mont_384: .align 5 _mul_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1385,7 +1385,7 @@ _mul_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1571,7 +1571,7 @@ __mul_384: .align 5 _mul_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1653,7 +1653,7 @@ _mul_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -1662,7 +1662,7 @@ _mul_382x: .align 5 _sqr_382x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1753,471 +1753,7 @@ _sqr_382x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - -.globl _sqr_mont_382x -.private_extern _sqr_mont_382x - -.align 5 -_sqr_mont_382x: -.long 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] // __mul_mont_384 wants them there - sub sp,sp,#112 // space for two 384-bit vectors + word - mov x4,x3 // adjust for missing b_ptr - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 // t0 = a->re + a->im - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 // t1 = a->re - a->im - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr // borrow flag as mask - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred // _mul_mont_384(ret->im, a->re, a->im) - - adds x19,x11,x11 // add with itself - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred // _mul_mont_384(ret->im, t0, t1) - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] // account for sign from a->re - a->im - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 - ret - - - -.align 5 -__mul_mont_383_nonred: - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] // pull r_ptr - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret @@ -2226,7 +1762,7 @@ __mul_mont_383_nonred: .align 5 _sgn0_pty_mont_384: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2271,7 +1807,7 @@ _sgn0_pty_mont_384: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret @@ -2280,7 +1816,7 @@ _sgn0_pty_mont_384: .align 5 _sgn0_pty_mont_384x: -.long 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2369,6 +1905,6 @@ _sgn0_pty_mont_384x: ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ -.long 3573752767 + hint #29 ret diff --git a/blst_src/build/mach-o/mulq_mont_384-x86_64.s b/blst_src/build/mach-o/mulq_mont_384-x86_64.s index 3cab12b1..b7c1ad8e 100644 --- a/blst_src/build/mach-o/mulq_mont_384-x86_64.s +++ b/blst_src/build/mach-o/mulq_mont_384-x86_64.s @@ -3101,765 +3101,3 @@ L$oop_sqr_383: #endif .cfi_endproc - -.p2align 5 -__mulq_mont_383_nonred: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - movq %rax,%rbp - mulq %r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r12 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - movq %r8,%r15 - imulq 8(%rsp),%r8 - - mulq %r13 - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r13 - - mulq 40(%rsi) - addq %rax,%r13 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r14 - - mulq 0(%rcx) - addq %rax,%r15 - movq %r8,%rax - adcq %rdx,%r15 - - mulq 8(%rcx) - addq %rax,%r9 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r9 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rcx) - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 24(%rcx) - addq %r15,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rcx) - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rcx) - addq %rax,%r13 - movq 8(%rbx),%rax - adcq $0,%rdx - addq %r15,%r13 - adcq %rdx,%r14 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 8(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r10 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 16(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r11 - adcq $0,%rdx - movq %rdx,%r15 - - movq %r9,%r8 - imulq 8(%rsp),%r9 - - mulq 24(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r12 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 32(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r15,%r13 - adcq $0,%rdx - movq %rdx,%r15 - - mulq 40(%rsi) - addq %r15,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r15 - - mulq 0(%rcx) - addq %rax,%r8 - movq %r9,%rax - adcq %rdx,%r8 - - mulq 8(%rcx) - addq %rax,%r10 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r10 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rcx) - addq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 24(%rcx) - addq %r8,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rcx) - addq %rax,%r13 - movq %r9,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rcx) - addq %rax,%r14 - movq 16(%rbx),%rax - adcq $0,%rdx - addq %r8,%r14 - adcq %rdx,%r15 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 8(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r11 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 16(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r12 - adcq $0,%rdx - movq %rdx,%r8 - - movq %r10,%r9 - imulq 8(%rsp),%r10 - - mulq 24(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r13 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 32(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r8,%r14 - adcq $0,%rdx - movq %rdx,%r8 - - mulq 40(%rsi) - addq %r8,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r8 - - mulq 0(%rcx) - addq %rax,%r9 - movq %r10,%rax - adcq %rdx,%r9 - - mulq 8(%rcx) - addq %rax,%r11 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r11 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rcx) - addq %rax,%r12 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 24(%rcx) - addq %r9,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rcx) - addq %rax,%r14 - movq %r10,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rcx) - addq %rax,%r15 - movq 24(%rbx),%rax - adcq $0,%rdx - addq %r9,%r15 - adcq %rdx,%r8 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 8(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r12 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 16(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r13 - adcq $0,%rdx - movq %rdx,%r9 - - movq %r11,%r10 - imulq 8(%rsp),%r11 - - mulq 24(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r14 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 32(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r9,%r15 - adcq $0,%rdx - movq %rdx,%r9 - - mulq 40(%rsi) - addq %r9,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r9 - - mulq 0(%rcx) - addq %rax,%r10 - movq %r11,%rax - adcq %rdx,%r10 - - mulq 8(%rcx) - addq %rax,%r12 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r12 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rcx) - addq %rax,%r13 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 24(%rcx) - addq %r10,%r14 - adcq $0,%rdx - addq %rax,%r14 - movq %r11,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rcx) - addq %rax,%r15 - movq %r11,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rcx) - addq %rax,%r8 - movq 32(%rbx),%rax - adcq $0,%rdx - addq %r10,%r8 - adcq %rdx,%r9 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 8(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r13 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 16(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r14 - adcq $0,%rdx - movq %rdx,%r10 - - movq %r12,%r11 - imulq 8(%rsp),%r12 - - mulq 24(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r15 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 32(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r10,%r8 - adcq $0,%rdx - movq %rdx,%r10 - - mulq 40(%rsi) - addq %r10,%r9 - adcq $0,%rdx - addq %rax,%r9 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq 0(%rcx) - addq %rax,%r11 - movq %r12,%rax - adcq %rdx,%r11 - - mulq 8(%rcx) - addq %rax,%r13 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r13 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rcx) - addq %rax,%r14 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 24(%rcx) - addq %r11,%r15 - adcq $0,%rdx - addq %rax,%r15 - movq %r12,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rcx) - addq %rax,%r8 - movq %r12,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rcx) - addq %rax,%r9 - movq 40(%rbx),%rax - adcq $0,%rdx - addq %r11,%r9 - adcq %rdx,%r10 - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 8(%rsi) - addq %rax,%r14 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r14 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 16(%rsi) - addq %rax,%r15 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r15 - adcq $0,%rdx - movq %rdx,%r11 - - movq %r13,%r12 - imulq 8(%rsp),%r13 - - mulq 24(%rsi) - addq %rax,%r8 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r8 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 32(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - addq %r11,%r9 - adcq $0,%rdx - movq %rdx,%r11 - - mulq 40(%rsi) - addq %r11,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq 0(%rcx) - addq %rax,%r12 - movq %r13,%rax - adcq %rdx,%r12 - - mulq 8(%rcx) - addq %rax,%r14 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r14 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 16(%rcx) - addq %rax,%r15 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r15 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 24(%rcx) - addq %r12,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r13,%rax - adcq $0,%rdx - movq %rdx,%r12 - - mulq 32(%rcx) - addq %rax,%r9 - movq %r13,%rax - adcq $0,%rdx - addq %r12,%r9 - adcq $0,%rdx - movq %rdx,%r12 - - mulq 40(%rcx) - addq %rax,%r10 - movq %r14,%rax - adcq $0,%rdx - addq %r12,%r10 - adcq %rdx,%r11 - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - -.globl _sqr_mont_382x -.private_extern _sqr_mont_382x - -.p2align 5 -_sqr_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -#ifdef __BLST_PORTABLE__ - testl $1,___blst_platform_cap(%rip) - jnz L$sqr_mont_382x$1 -#endif - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rsi,16(%rsp) - movq %rdi,24(%rsp) - - - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rax - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%r12 - movq 24(%rsi),%r13 - - movq 24(%rsp),%rdi - call __mulq_mont_383_nonred - addq %r14,%r14 - adcq %r15,%r15 - adcq %r8,%r8 - adcq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - - movq %r14,48(%rdi) - movq %r15,56(%rdi) - movq %r8,64(%rdi) - movq %r9,72(%rdi) - movq %r10,80(%rdi) - movq %r11,88(%rdi) - - leaq 32(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rax - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%r12 - movq 32+24(%rsp),%r13 - - call __mulq_mont_383_nonred - movq 32+96(%rsp),%rsi - movq 32+0(%rsp),%r12 - movq 32+8(%rsp),%r13 - andq %rsi,%r12 - movq 32+16(%rsp),%rax - andq %rsi,%r13 - movq 32+24(%rsp),%rbx - andq %rsi,%rax - movq 32+32(%rsp),%rbp - andq %rsi,%rbx - andq %rsi,%rbp - andq 32+40(%rsp),%rsi - - subq %r12,%r14 - movq 0(%rcx),%r12 - sbbq %r13,%r15 - movq 8(%rcx),%r13 - sbbq %rax,%r8 - movq 16(%rcx),%rax - sbbq %rbx,%r9 - movq 24(%rcx),%rbx - sbbq %rbp,%r10 - movq 32(%rcx),%rbp - sbbq %rsi,%r11 - sbbq %rsi,%rsi - - andq %rsi,%r12 - andq %rsi,%r13 - andq %rsi,%rax - andq %rsi,%rbx - andq %rsi,%rbp - andq 40(%rcx),%rsi - - addq %r12,%r14 - adcq %r13,%r15 - adcq %rax,%r8 - adcq %rbx,%r9 - adcq %rbp,%r10 - adcq %rsi,%r11 - - movq %r14,0(%rdi) - movq %r15,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq %r10,32(%rdi) - movq %r11,40(%rdi) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - diff --git a/blst_src/build/mach-o/mulx_mont_256-x86_64.s b/blst_src/build/mach-o/mulx_mont_256-x86_64.s index 9f0c6a2b..df73cc42 100644 --- a/blst_src/build/mach-o/mulx_mont_256-x86_64.s +++ b/blst_src/build/mach-o/mulx_mont_256-x86_64.s @@ -202,17 +202,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 16(%rbx),%rdx adcxq %rbp,%r13 - adoxq %r9,%r14 - adcxq %r10,%r14 - adoxq %r10,%r15 + adoxq %r10,%r9 + adcxq %r9,%r14 adcxq %r10,%r15 - adoxq %r10,%r10 - adcq $0,%r10 movq %rax,%r11 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r10,%r10 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r12 adcxq %r9,%r13 @@ -247,17 +244,14 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq 24(%rbx),%rdx adcxq %rbp,%r14 - adoxq %r9,%r15 - adcxq %r11,%r15 - adoxq %r11,%r10 + adoxq %r11,%r9 + adcxq %r9,%r15 adcxq %r11,%r10 - adoxq %r11,%r11 - adcq $0,%r11 movq %rax,%r12 imulq %r8,%rax - xorq %rbp,%rbp + xorq %r11,%r11 mulxq 0+128(%rsi),%rbp,%r9 adoxq %rbp,%r13 adcxq %r9,%r14 @@ -292,16 +286,13 @@ __mulx_mont_sparse_256: mulxq 24+128(%rcx),%rbp,%r9 movq %rax,%rdx adcxq %rbp,%r15 - adoxq %r9,%r10 - adcxq %r12,%r10 - adoxq %r12,%r11 + adoxq %r12,%r9 + adcxq %r9,%r10 adcxq %r12,%r11 - adoxq %r12,%r12 - adcq $0,%r12 imulq %r8,%rdx - xorq %rbp,%rbp + xorq %r12,%r12 mulxq 0+128(%rcx),%r13,%r9 adcxq %rax,%r13 adoxq %r9,%r14 @@ -318,11 +309,10 @@ __mulx_mont_sparse_256: movq %r14,%rdx leaq 128(%rcx),%rcx adcxq %rbp,%r10 - adoxq %r9,%r11 + adoxq %r13,%r9 movq %r15,%rax - adcxq %r13,%r11 - adoxq %r13,%r12 - adcq $0,%r12 + adcxq %r9,%r11 + adcxq %r13,%r12 diff --git a/blst_src/build/mach-o/mulx_mont_384-x86_64.s b/blst_src/build/mach-o/mulx_mont_384-x86_64.s index 45b3600e..e32c8834 100644 --- a/blst_src/build/mach-o/mulx_mont_384-x86_64.s +++ b/blst_src/build/mach-o/mulx_mont_384-x86_64.s @@ -2028,8 +2028,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r8,%rdx adoxq %rdi,%r14 - adcxq %rbp,%r15 - adoxq %rax,%r15 + adcxq %rax,%rbp + adoxq %rbp,%r15 adoxq %rax,%rax @@ -2057,11 +2057,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %r8,%r14 - adoxq %r8,%r15 + adoxq %r8,%rbp + adcxq %rbp,%r14 adcxq %r8,%r15 - adoxq %r8,%rax adcxq %r8,%rax movq %r9,16(%rsp) imulq 8(%rsp),%r9 @@ -2091,8 +2089,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r9,%rdx adoxq %rdi,%r15 - adcxq %rbp,%rax - adoxq %r8,%rax + adcxq %r8,%rbp + adoxq %rbp,%rax adoxq %r8,%r8 @@ -2120,11 +2118,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r9,%r15 - adoxq %r9,%rax + adoxq %r9,%rbp + adcxq %rbp,%r15 adcxq %r9,%rax - adoxq %r9,%r8 adcxq %r9,%r8 movq %r10,16(%rsp) imulq 8(%rsp),%r10 @@ -2154,8 +2150,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r10,%rdx adoxq %rdi,%rax - adcxq %rbp,%r8 - adoxq %r9,%r8 + adcxq %r9,%rbp + adoxq %rbp,%r8 adoxq %r9,%r9 @@ -2183,11 +2179,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r10,%rax - adoxq %r10,%r8 + adoxq %r10,%rbp + adcxq %rbp,%rax adcxq %r10,%r8 - adoxq %r10,%r9 adcxq %r10,%r9 movq %r11,16(%rsp) imulq 8(%rsp),%r11 @@ -2217,8 +2211,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r11,%rdx adoxq %rdi,%r8 - adcxq %rbp,%r9 - adoxq %r10,%r9 + adcxq %r10,%rbp + adoxq %rbp,%r9 adoxq %r10,%r10 @@ -2246,11 +2240,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r11,%r8 - adoxq %r11,%r9 + adoxq %r11,%rbp + adcxq %rbp,%r8 adcxq %r11,%r9 - adoxq %r11,%r10 adcxq %r11,%r10 movq %r12,16(%rsp) imulq 8(%rsp),%r12 @@ -2280,8 +2272,8 @@ __mulx_mont_384: mulxq 40+128(%rsi),%rdi,%rbp movq %r12,%rdx adoxq %rdi,%r9 - adcxq %rbp,%r10 - adoxq %r11,%r10 + adcxq %r11,%rbp + adoxq %rbp,%r10 adoxq %r11,%r11 @@ -2309,11 +2301,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r12,%r9 - adoxq %r12,%r10 + adoxq %r12,%rbp + adcxq %rbp,%r9 adcxq %r12,%r10 - adoxq %r12,%r11 adcxq %r12,%r11 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -2344,10 +2334,9 @@ __mulx_mont_384: mulxq 40+128(%rcx),%rdi,%rbp adcxq %rdi,%r9 - adoxq %rbp,%r10 + adoxq %r12,%rbp movq %r14,%rdx - adcxq %r12,%r10 - adoxq %r12,%r11 + adcxq %rbp,%r10 leaq 128(%rcx),%rcx movq %r8,%r12 adcq $0,%r11 @@ -2735,9 +2724,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 16(%rbx),%rdx adcxq %rdi,%r13 - adoxq %rbp,%r14 - adcxq %rax,%r14 - adoxq %rax,%r15 + adoxq %rax,%rbp + adcxq %rbp,%r14 adcxq %rax,%r15 movq %r9,%r8 imulq 8(%rsp),%r9 @@ -2795,9 +2783,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 24(%rbx),%rdx adcxq %rdi,%r14 - adoxq %rbp,%r15 - adcxq %r8,%r15 - adoxq %r8,%rax + adoxq %r8,%rbp + adcxq %rbp,%r15 adcxq %r8,%rax movq %r10,%r9 imulq 8(%rsp),%r10 @@ -2855,9 +2842,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 32(%rbx),%rdx adcxq %rdi,%r15 - adoxq %rbp,%rax - adcxq %r9,%rax - adoxq %r9,%r8 + adoxq %r9,%rbp + adcxq %rbp,%rax adcxq %r9,%r8 movq %r11,%r10 imulq 8(%rsp),%r11 @@ -2915,9 +2901,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq 40(%rbx),%rdx adcxq %rdi,%rax - adoxq %rbp,%r8 - adcxq %r10,%r8 - adoxq %r10,%r9 + adoxq %r10,%rbp + adcxq %rbp,%r8 adcxq %r10,%r9 movq %r12,%r11 imulq 8(%rsp),%r12 @@ -2975,9 +2960,8 @@ __mulx_mont_383_nonred: mulxq 40+128(%rcx),%rdi,%rbp movq %r13,%rdx adcxq %rdi,%r8 - adoxq %rbp,%r9 - adcxq %r11,%r9 - adoxq %r11,%r10 + adoxq %r11,%rbp + adcxq %rbp,%r9 adcxq %r11,%r10 imulq 8(%rsp),%rdx movq 24(%rsp),%rbx @@ -3031,206 +3015,3 @@ __mulx_mont_383_nonred: #endif .cfi_endproc -.globl _sqrx_mont_382x -.private_extern _sqrx_mont_382x - -.p2align 5 -_sqrx_mont_382x: -.cfi_startproc - .byte 0xf3,0x0f,0x1e,0xfa - - -L$sqr_mont_382x$1: - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $136,%rsp -.cfi_adjust_cfa_offset 136 - - - movq %rcx,0(%rsp) - movq %rdx,%rcx - movq %rdi,16(%rsp) - movq %rsi,24(%rsp) - - -#ifdef __SGX_LVI_HARDENING__ - lfence -#endif - movq 0(%rsi),%r8 - movq 8(%rsi),%r9 - movq 16(%rsi),%r10 - movq 24(%rsi),%r11 - movq 32(%rsi),%r12 - movq 40(%rsi),%r13 - - movq %r8,%r14 - addq 48(%rsi),%r8 - movq %r9,%r15 - adcq 56(%rsi),%r9 - movq %r10,%rax - adcq 64(%rsi),%r10 - movq %r11,%rdx - adcq 72(%rsi),%r11 - movq %r12,%rbx - adcq 80(%rsi),%r12 - movq %r13,%rbp - adcq 88(%rsi),%r13 - - subq 48(%rsi),%r14 - sbbq 56(%rsi),%r15 - sbbq 64(%rsi),%rax - sbbq 72(%rsi),%rdx - sbbq 80(%rsi),%rbx - sbbq 88(%rsi),%rbp - sbbq %rdi,%rdi - - movq %r8,32+0(%rsp) - movq %r9,32+8(%rsp) - movq %r10,32+16(%rsp) - movq %r11,32+24(%rsp) - movq %r12,32+32(%rsp) - movq %r13,32+40(%rsp) - - movq %r14,32+48(%rsp) - movq %r15,32+56(%rsp) - movq %rax,32+64(%rsp) - movq %rdx,32+72(%rsp) - movq %rbx,32+80(%rsp) - movq %rbp,32+88(%rsp) - movq %rdi,32+96(%rsp) - - - - leaq 48(%rsi),%rbx - - movq 48(%rsi),%rdx - movq 0(%rsi),%r14 - movq 8(%rsi),%r15 - movq 16(%rsi),%rax - movq 24(%rsi),%r12 - movq 32(%rsi),%rdi - movq 40(%rsi),%rbp - leaq -128(%rsi),%rsi - leaq -128(%rcx),%rcx - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - addq %rdx,%rdx - adcq %r15,%r15 - adcq %rax,%rax - adcq %r12,%r12 - adcq %rdi,%rdi - adcq %rbp,%rbp - - movq %rdx,48(%rbx) - movq %r15,56(%rbx) - movq %rax,64(%rbx) - movq %r12,72(%rbx) - movq %rdi,80(%rbx) - movq %rbp,88(%rbx) - - leaq 32-128(%rsp),%rsi - leaq 32+48(%rsp),%rbx - - movq 32+48(%rsp),%rdx - movq 32+0(%rsp),%r14 - movq 32+8(%rsp),%r15 - movq 32+16(%rsp),%rax - movq 32+24(%rsp),%r12 - movq 32+32(%rsp),%rdi - movq 32+40(%rsp),%rbp - - - - mulxq %r14,%r8,%r9 - call __mulx_mont_383_nonred - movq 32+96(%rsp),%r14 - leaq 128(%rcx),%rcx - movq 32+0(%rsp),%r8 - andq %r14,%r8 - movq 32+8(%rsp),%r9 - andq %r14,%r9 - movq 32+16(%rsp),%r10 - andq %r14,%r10 - movq 32+24(%rsp),%r11 - andq %r14,%r11 - movq 32+32(%rsp),%r13 - andq %r14,%r13 - andq 32+40(%rsp),%r14 - - subq %r8,%rdx - movq 0(%rcx),%r8 - sbbq %r9,%r15 - movq 8(%rcx),%r9 - sbbq %r10,%rax - movq 16(%rcx),%r10 - sbbq %r11,%r12 - movq 24(%rcx),%r11 - sbbq %r13,%rdi - movq 32(%rcx),%r13 - sbbq %r14,%rbp - sbbq %r14,%r14 - - andq %r14,%r8 - andq %r14,%r9 - andq %r14,%r10 - andq %r14,%r11 - andq %r14,%r13 - andq 40(%rcx),%r14 - - addq %r8,%rdx - adcq %r9,%r15 - adcq %r10,%rax - adcq %r11,%r12 - adcq %r13,%rdi - adcq %r14,%rbp - - movq %rdx,0(%rbx) - movq %r15,8(%rbx) - movq %rax,16(%rbx) - movq %r12,24(%rbx) - movq %rdi,32(%rbx) - movq %rbp,40(%rbx) - leaq 136(%rsp),%r8 - movq 0(%r8),%r15 -.cfi_restore %r15 - movq 8(%r8),%r14 -.cfi_restore %r14 - movq 16(%r8),%r13 -.cfi_restore %r13 - movq 24(%r8),%r12 -.cfi_restore %r12 - movq 32(%r8),%rbx -.cfi_restore %rbx - movq 40(%r8),%rbp -.cfi_restore %rbp - leaq 48(%r8),%rsp -.cfi_adjust_cfa_offset -136-8*6 - - -#ifdef __SGX_LVI_HARDENING__ - popq %rdx - lfence - jmpq *%rdx - ud2 -#else - .byte 0xf3,0xc3 -#endif -.cfi_endproc - diff --git a/blst_src/build/mach-o/sha256-armv8.S b/blst_src/build/mach-o/sha256-armv8.S index cf0f80d1..c7befef9 100644 --- a/blst_src/build/mach-o/sha256-armv8.S +++ b/blst_src/build/mach-o/sha256-armv8.S @@ -47,6 +47,7 @@ LK256: .align 6 _blst_sha256_block_armv8: + hint #34 Lv8_entry: stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -187,6 +188,7 @@ Loop_hw: .align 4 _blst_sha256_block_data_order: + hint #34 adrp x16,___blst_platform_cap@PAGE ldr w16,[x16,___blst_platform_cap@PAGEOFF] tst w16,#1 @@ -1036,6 +1038,7 @@ L_00_48: .align 4 _blst_sha256_emit: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] #ifndef __AARCH64EB__ @@ -1064,6 +1067,7 @@ _blst_sha256_emit: .align 4 _blst_sha256_bcopy: + hint #34 Loop_bcopy: ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1077,6 +1081,7 @@ Loop_bcopy: .align 4 _blst_sha256_hcopy: + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/build/win64/add_mod_256-armv8.asm b/blst_src/build/win64/add_mod_256-armv8.asm index 91b7b5a7..ea64176f 100644 --- a/blst_src/build/win64/add_mod_256-armv8.asm +++ b/blst_src/build/win64/add_mod_256-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_256|[FUNC] ALIGN 32 |add_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x12,x13,[x2] @@ -41,6 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_256|[FUNC] ALIGN 32 |mul_by_3_mod_256| PROC + hint #34 ldp x12,x13,[x1] ldp x14,x15,[x1,#16] @@ -90,6 +92,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |lshift_mod_256|[FUNC] ALIGN 32 |lshift_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] @@ -128,6 +131,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |rshift_mod_256|[FUNC] ALIGN 32 |rshift_mod_256| PROC + hint #34 ldp x8,x9,[x1] ldp x10,x11,[x1,#16] diff --git a/blst_src/build/win64/add_mod_384-armv8.asm b/blst_src/build/win64/add_mod_384-armv8.asm index 5b2989ce..dfb0a747 100644 --- a/blst_src/build/win64/add_mod_384-armv8.asm +++ b/blst_src/build/win64/add_mod_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384|[FUNC] ALIGN 32 |add_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -27,7 +27,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -73,7 +73,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384x|[FUNC] ALIGN 32 |add_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -101,7 +101,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -110,7 +110,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |rshift_mod_384|[FUNC] ALIGN 32 |rshift_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -137,7 +137,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -172,7 +172,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |div_by_2_mod_384|[FUNC] ALIGN 32 |div_by_2_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -196,7 +196,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -205,7 +205,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |lshift_mod_384|[FUNC] ALIGN 32 |lshift_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -232,7 +232,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -270,7 +270,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_384|[FUNC] ALIGN 32 |mul_by_3_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -300,7 +300,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -309,7 +309,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_8_mod_384|[FUNC] ALIGN 32 |mul_by_8_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -335,7 +335,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -344,7 +344,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_3_mod_384x|[FUNC] ALIGN 32 |mul_by_3_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -389,7 +389,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -398,7 +398,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_8_mod_384x|[FUNC] ALIGN 32 |mul_by_8_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -435,7 +435,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -444,7 +444,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |cneg_mod_384|[FUNC] ALIGN 32 |cneg_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -486,7 +486,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -495,7 +495,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384|[FUNC] ALIGN 32 |sub_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -515,7 +515,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -558,7 +558,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384x|[FUNC] ALIGN 32 |sub_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -586,7 +586,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -595,7 +595,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_by_1_plus_i_mod_384x|[FUNC] ALIGN 32 |mul_by_1_plus_i_mod_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -628,7 +628,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -637,6 +637,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mod_384|[FUNC] ALIGN 32 |sgn0_pty_mod_384| PROC + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -674,6 +675,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mod_384x|[FUNC] ALIGN 32 |sgn0_pty_mod_384x| PROC + hint #34 ldp x10,x11,[x0] ldp x12,x13,[x0,#16] ldp x14,x15,[x0,#32] @@ -755,14 +757,14 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_32|[FUNC] ALIGN 32 |vec_select_32| PROC + hint #34 dup v6.2d, x3 - ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 + ld1 {v0.2d, v1.2d}, [x1] cmeq v6.2d, v6.2d, #0 - ld1 {v3.2d, v4.2d, v5.2d}, [x2],#48 + ld1 {v3.2d, v4.2d}, [x2] bit v0.16b, v3.16b, v6.16b bit v1.16b, v4.16b, v6.16b - bit v2.16b, v5.16b, v6.16b - st1 {v0.2d, v1.2d, v2.2d}, [x0] + st1 {v0.2d, v1.2d}, [x0] ret ENDP @@ -770,6 +772,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_48|[FUNC] ALIGN 32 |vec_select_48| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -785,6 +788,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_96|[FUNC] ALIGN 32 |vec_select_96| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -806,6 +810,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_192|[FUNC] ALIGN 32 |vec_select_192| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -839,6 +844,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_144|[FUNC] ALIGN 32 |vec_select_144| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -866,6 +872,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_select_288|[FUNC] ALIGN 32 |vec_select_288| PROC + hint #34 dup v6.2d, x3 ld1 {v0.2d, v1.2d, v2.2d}, [x1],#48 cmeq v6.2d, v6.2d, #0 @@ -911,6 +918,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_prefetch|[FUNC] ALIGN 32 |vec_prefetch| PROC + hint #34 add x1, x1, x0 sub x1, x1, #1 mov x2, #64 @@ -951,6 +959,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_is_zero_16x|[FUNC] ALIGN 32 |vec_is_zero_16x| PROC + hint #34 ld1 {v0.2d}, [x0], #16 lsr x1, x1, #4 sub x1, x1, #1 @@ -976,6 +985,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |vec_is_equal_16x|[FUNC] ALIGN 32 |vec_is_equal_16x| PROC + hint #34 ld1 {v0.2d}, [x0], #16 ld1 {v1.2d}, [x1], #16 lsr x2, x2, #4 diff --git a/blst_src/build/win64/blst.def b/blst_src/build/win64/blst.def index dda95336..5bc8602e 100644 --- a/blst_src/build/win64/blst.def +++ b/blst_src/build/win64/blst.def @@ -217,5 +217,7 @@ EXPORTS blst_p2_sizeof blst_p2_affine_sizeof blst_fp12_sizeof + blst_fp_from_le_bytes + blst_fp_from_be_bytes blst_sha256 diff --git a/blst_src/build/win64/ct_inverse_mod_256-armv8.asm b/blst_src/build/win64/ct_inverse_mod_256-armv8.asm index fde83e0e..b6be3bc2 100644 --- a/blst_src/build/win64/ct_inverse_mod_256-armv8.asm +++ b/blst_src/build/win64/ct_inverse_mod_256-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |ct_inverse_mod_256|[FUNC] ALIGN 32 |ct_inverse_mod_256| PROC - DCDU 3573752639 + hint #25 stp x29, x30, [sp,#-10*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -22,6 +22,7 @@ __SIZEOF_POINTER__ SETA 64/8 if :def: __CHERI_PURE_CAPABILITY__ add x1,sp,#16+511 alignd c1,c1,#9 + scbnds c1,c1,#512 else add x1, sp, #16+511 and x1, x1, #-512 @@ -41,7 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 str x12,[x0,#8*8] @@ -50,18 +51,18 @@ __SIZEOF_POINTER__ SETA 64/8 mov x13, x15 add x0,x0,#8*4 bl __smul_256_n_shift_by_31 - str x12, [x0,#8*9] + str x12, [x0,#8*10] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -73,29 +74,27 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256_n_shift_by_31 ldr x8, [x1,#8*8] - ldr x9, [x1,#8*13] + ldr x9, [x1,#8*14] madd x4, x16, x8, xzr madd x4, x17, x9, x4 - str x4, [x0,#8*4] asr x5, x4, #63 - stp x5, x5, [x0,#8*5] - stp x5, x5, [x0,#8*7] + stp x4, x5, [x0,#8*4] + stp x5, x5, [x0,#8*6] madd x4, x12, x8, xzr madd x4, x13, x9, x4 - str x4, [x0,#8*9] asr x5, x4, #63 - stp x5, x5, [x0,#8*10] + stp x4, x5, [x0,#8*10] stp x5, x5, [x0,#8*12] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -108,25 +107,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -139,25 +132,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -170,25 +157,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -201,25 +182,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -232,25 +207,19 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -263,25 +232,24 @@ __SIZEOF_POINTER__ SETA 64/8 add x0,x0,#8*4 bl __smul_256x63 - adc x22, x22, x23 - str x22, [x0,#8*4] - + asr x24, x24, #63 + str x24, [x0,#8*4] mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 - adc x22, x22, x23 - stp x22, x22, [x0,#8*4] - stp x22, x22, [x0,#8*6] + asr x24, x24, #63 + stp x24, x24, [x0,#8*4] + stp x24, x24, [x0,#8*6] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -296,21 +264,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -325,21 +292,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -354,21 +320,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -383,21 +348,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -412,21 +376,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -441,21 +404,20 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif bl __ab_approximation_31_256 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif bl __smul_256_n_shift_by_31 mov x16, x12 @@ -470,16 +432,15 @@ __SIZEOF_POINTER__ SETA 64/8 bl __smul_256x63 adc x22, x22, x23 str x22, [x0,#8*4] - mov x16, x12 mov x17, x13 - add x0,x0,#8*5 + add x0,x0,#8*6 bl __smul_256x63 bl __smul_512x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #47 @@ -543,7 +504,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x23, x24, [x29,#6*__SIZEOF_POINTER__] ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldr x29, [sp],#10*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -581,11 +542,11 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x6, x6, x20 adcs x24, x24, x21 adc x26, xzr, xzr - ldp x8, x9, [x1,#8*0+104] + ldp x8, x9, [x1,#8*0+112] asr x14, x17, #63 - ldp x10, x11, [x1,#8*2+104] + ldp x10, x11, [x1,#8*2+112] eor x17, x17, x14 - ldr x23, [x1,#8*4+104] + ldr x23, [x1,#8*4+112] eor x8, x8, x14 sub x17, x17, x14 @@ -627,9 +588,9 @@ __SIZEOF_POINTER__ SETA 64/8 ALIGN 32 |__smul_512x63_tail| PROC umulh x24, x7, x16 - ldp x5, x6, [x1,#8*18] + ldr x5, [x1,#8*19] adc x26, x26, xzr - ldr x7, [x1,#8*20] + ldp x6, x7, [x1,#8*20] and x22, x22, x16 umulh x11, x11, x17 diff --git a/blst_src/build/win64/ct_inverse_mod_384-armv8.asm b/blst_src/build/win64/ct_inverse_mod_384-armv8.asm index 36fe37b0..be929b12 100644 --- a/blst_src/build/win64/ct_inverse_mod_384-armv8.asm +++ b/blst_src/build/win64/ct_inverse_mod_384-armv8.asm @@ -4,10 +4,10 @@ __SIZEOF_POINTER__ SETA 64/8 - EXPORT |ct_inverse_mod_383|[FUNC] + EXPORT |ct_inverse_mod_384|[FUNC] ALIGN 32 -|ct_inverse_mod_383| PROC - DCDU 3573752639 +|ct_inverse_mod_384| PROC + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -24,6 +24,7 @@ __SIZEOF_POINTER__ SETA 64/8 if :def: __CHERI_PURE_CAPABILITY__ add x1,sp,#32+511 alignd c1,c1,#9 + scbnds c1,c1,#512 else add x1, sp, #32+511 and x1, x1, #-512 @@ -47,40 +48,40 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 str x15,[x0,#8*12] mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 - str x15, [x0,#8*12] + bl __smul_384_n_shift_by_62 + str x15, [x0,#8*14] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 ldr x7, [x1,#8*12] - ldr x8, [x1,#8*18] + ldr x8, [x1,#8*20] mul x3, x20, x7 smulh x4, x20, x7 mul x5, x21, x8 @@ -98,266 +99,269 @@ __SIZEOF_POINTER__ SETA 64/8 smulh x6, x16, x8 adds x3, x3, x5 adc x4, x4, x6 - stp x3, x4, [x0,#8*12] + stp x3, x4, [x0,#8*14] asr x5, x4, #63 - stp x5, x5, [x0,#8*14] stp x5, x5, [x0,#8*16] + stp x5, x5, [x0,#8*18] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + asr x27, x27, #63 + str x27, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 + add x0,x0,#8*8 + bl __smul_384x63 asr x27, x27, #63 stp x27, x27, [x0,#8*6] stp x27, x27, [x0,#8*8] stp x27, x27, [x0,#8*10] eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 bl __ab_approximation_62 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 mov x20, x15 mov x21, x16 mov x15, x17 mov x16, x19 add x0,x0,#8*6 - bl __smul_383_n_shift_by_62 + bl __smul_384_n_shift_by_62 add x0,x0,#8*6 - bl __smul_383x63 - + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif mov x2, #62 @@ -367,7 +371,7 @@ __SIZEOF_POINTER__ SETA 64/8 eor x0, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c0,csp,x0 + scvalue c0,c1,x0 endif str x3, [x0,#8*0] str x9, [x0,#8*6] @@ -377,20 +381,22 @@ __SIZEOF_POINTER__ SETA 64/8 mov x15, x17 mov x16, x19 add x0,x0,#8*12 - bl __smul_383x63 + bl __smul_384x63 + adc x25, x25, x26 + str x25, [x0,#8*6] mov x20, x15 mov x21, x16 - add x0,x0,#8*6 - bl __smul_383x63 - bl __smul_767x63_tail + add x0,x0,#8*8 + bl __smul_384x63 + bl __smul_768x63_tail eor x1, x1, #256 if :def: __CHERI_PURE_CAPABILITY__ - scvalue c1,csp,x1 + scvalue c1,c0,x1 endif - mov x2, #22 + mov x2, #24 ldr x3, [x1,#8*0] eor x8, x8, x8 @@ -401,25 +407,60 @@ __SIZEOF_POINTER__ SETA 64/8 mov x20, x17 mov x21, x19 ldp x0, x15, [sp] - bl __smul_383x63 - bl __smul_767x63_tail + bl __smul_384x63 + bl __smul_768x63_tail ldr x30, [x29,#__SIZEOF_POINTER__] - asr x22, x8, #63 + smulh x23, x8, x21 + adc x26, x26, x28 ldp x9, x10, [x15,#8*0] + add x23, x23, x26 ldp x11, x12, [x15,#8*2] + asr x22, x23, #63 ldp x13, x14, [x15,#8*4] - and x9, x9, x22 - and x10, x10, x22 + and x26, x9, x22 + and x27, x10, x22 + adds x3, x3, x26 + and x28, x11, x22 + adcs x4, x4, x27 + and x2, x12, x22 + adcs x5, x5, x28 + and x26, x13, x22 + adcs x6, x6, x2 + and x27, x14, x22 + adcs x7, x7, x26 + adcs x8, x25, x27 + adc x23, x23, xzr + + neg x22, x23 + orr x23, x23, x22 + asr x22, x22, #63 + + and x9, x9, x23 + and x10, x10, x23 + and x11, x11, x23 + and x12, x12, x23 + and x13, x13, x23 + and x14, x14, x23 + + eor x9, x9, x22 + eor x10, x10, x22 + adds x9, x9, x22, lsr#63 + eor x11, x11, x22 + adcs x10, x10, xzr + eor x12, x12, x22 + adcs x11, x11, xzr + eor x13, x13, x22 + adcs x12, x12, xzr + eor x14, x14, x22 + adcs x13, x13, xzr + adc x14, x14, xzr + adds x3, x3, x9 - and x11, x11, x22 adcs x4, x4, x10 - and x12, x12, x22 adcs x5, x5, x11 - and x13, x13, x22 adcs x6, x6, x12 - and x14, x14, x22 stp x3, x4, [x0,#8*6] adcs x7, x7, x13 stp x5, x6, [x0,#8*8] @@ -433,7 +474,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -441,7 +482,7 @@ __SIZEOF_POINTER__ SETA 64/8 ALIGN 32 -|__smul_383x63| PROC +|__smul_384x63| PROC ldp x3, x4, [x1,#8*0+96] asr x17, x20, #63 ldp x5, x6, [x1,#8*2+96] @@ -449,6 +490,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x7, x8, [x1,#8*4+96] eor x3, x3, x17 + ldr x25, [x1,#8*6+96] sub x20, x20, x17 eor x4, x4, x17 adds x3, x3, x17, lsr#63 @@ -463,28 +505,33 @@ __SIZEOF_POINTER__ SETA 64/8 umulh x23, x4, x20 adcs x7, x7, xzr umulh x24, x5, x20 - adcs x8, x8, xzr - umulh x25, x6, x20 - umulh x26, x7, x20 + eor x25, x25, x17 mul x3, x3, x20 + adcs x8, x8, xzr mul x4, x4, x20 + adcs x25, x25, xzr + cmp x20, #0 mul x5, x5, x20 + cselne x25,x25,xzr adds x4, x4, x22 - mul x6, x6, x20 + umulh x22, x6, x20 adcs x5, x5, x23 + umulh x23, x7, x20 + mul x6, x6, x20 mul x7, x7, x20 adcs x6, x6, x24 mul x27,x8, x20 - adcs x7, x7, x25 - adcs x27,x27,x26 + adcs x7, x7, x22 + adcs x27,x27,x23 adc x2, xzr, xzr - ldp x9, x10, [x1,#8*0+144] + ldp x9, x10, [x1,#8*0+160] asr x17, x21, #63 - ldp x11, x12, [x1,#8*2+144] + ldp x11, x12, [x1,#8*2+160] eor x21, x21, x17 - ldp x13, x14, [x1,#8*4+144] + ldp x13, x14, [x1,#8*4+160] eor x9, x9, x17 + ldr x26, [x1,#8*6+160] sub x21, x21, x17 eor x10, x10, x17 adds x9, x9, x17, lsr#63 @@ -499,21 +546,25 @@ __SIZEOF_POINTER__ SETA 64/8 umulh x23, x10, x21 adcs x13, x13, xzr umulh x24, x11, x21 - adcs x14, x14, xzr - umulh x25, x12, x21 - adc x19, xzr, xzr - umulh x26, x13, x21 + eor x26, x26, x17 mul x9, x9, x21 + adcs x14, x14, xzr mul x10, x10, x21 + adcs x26, x26, xzr + adc x19, xzr, xzr + cmp x21, #0 mul x11, x11, x21 + cselne x26,x26,xzr adds x10, x10, x22 - mul x12, x12, x21 + umulh x22, x12, x21 adcs x11, x11, x23 + umulh x23, x13, x21 + mul x12, x12, x21 mul x13, x13, x21 adcs x12, x12, x24 mul x28,x14, x21 - adcs x13, x13, x25 - adcs x28,x28,x26 + adcs x13, x13, x22 + adcs x28,x28,x23 adc x2, x2, xzr adds x3, x3, x9 @@ -525,41 +576,41 @@ __SIZEOF_POINTER__ SETA 64/8 stp x5, x6, [x0,#8*2] adcs x27, x27, x28 stp x7, x27, [x0,#8*4] - adc x28, x2, xzr ret ENDP ALIGN 32 -|__smul_767x63_tail| PROC - smulh x27, x8, x20 - ldp x3, x4, [x1,#8*24] - umulh x14,x14, x21 - ldp x5, x6, [x1,#8*26] - ldp x7, x8, [x1,#8*28] +|__smul_768x63_tail| PROC + umulh x27, x8, x20 + ldr x4, [x1,#8*27] + adc x2, x2, xzr + ldp x5, x6, [x1,#8*28] + and x25, x25, x20 + ldp x7, x8, [x1,#8*30] + sub x27, x27, x25 - eor x3, x3, x17 + umulh x14, x14, x21 eor x4, x4, x17 eor x5, x5, x17 - adds x3, x3, x19 eor x6, x6, x17 - adcs x4, x4, xzr + adds x4, x4, x19 eor x7, x7, x17 adcs x5, x5, xzr eor x8, x8, x17 adcs x6, x6, xzr - umulh x22, x3, x21 + umulh x22, x26, x21 adcs x7, x7, xzr umulh x23, x4, x21 adc x8, x8, xzr umulh x24, x5, x21 - add x14, x14, x28 + add x14, x14, x2 umulh x25, x6, x21 asr x28, x27, #63 - umulh x26, x7, x21 - mul x3, x3, x21 + umulh x2, x7, x21 + mul x3, x26, x21 mul x4, x4, x21 mul x5, x5, x21 adds x3, x3, x14 @@ -567,10 +618,11 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x4, x4, x22 mul x7, x7, x21 adcs x5, x5, x23 - mul x8, x8, x21 + mul x22, x8, x21 adcs x6, x6, x24 adcs x7, x7, x25 - adc x8, x8, x26 + adcs x25, x22, x2 + adc x26, xzr, xzr adds x3, x3, x27 adcs x4, x4, x28 @@ -579,15 +631,15 @@ __SIZEOF_POINTER__ SETA 64/8 stp x3, x4, [x0,#8*6] adcs x7, x7, x28 stp x5, x6, [x0,#8*8] - adc x8, x8, x28 - stp x7, x8, [x0,#8*10] + adcs x25, x25, x28 + stp x7, x25, [x0,#8*10] ret ENDP ALIGN 32 -|__smul_383_n_shift_by_62| PROC +|__smul_384_n_shift_by_62| PROC ldp x3, x4, [x1,#8*0+0] asr x28, x15, #63 ldp x5, x6, [x1,#8*2+0] @@ -607,25 +659,27 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x6, x6, xzr umulh x23, x4, x2 eor x8, x8, x28 - umulh x24, x5, x2 + mul x3, x3, x2 adcs x7, x7, xzr - umulh x25, x6, x2 + mul x4, x4, x2 adc x8, x8, xzr - umulh x26, x7, x2 - smulh x27, x8, x2 - mul x3, x3, x2 - mul x4, x4, x2 - mul x5, x5, x2 + umulh x24, x5, x2 + and x28, x28, x2 + umulh x25, x6, x2 adds x4, x4, x22 + mul x5, x5, x2 + umulh x22, x7, x2 + neg x28, x28 mul x6, x6, x2 adcs x5, x5, x23 + umulh x23, x8, x2 mul x7, x7, x2 adcs x6, x6, x24 mul x8, x8, x2 adcs x7, x7, x25 - adcs x8, x8 ,x26 - adc x27, x27, xzr + adcs x8, x8, x22 + adc x27, x23, x28 ldp x9, x10, [x1,#8*0+48] asr x28, x16, #63 ldp x11, x12, [x1,#8*2+48] @@ -645,25 +699,27 @@ __SIZEOF_POINTER__ SETA 64/8 adcs x12, x12, xzr umulh x23, x10, x2 eor x14, x14, x28 - umulh x24, x11, x2 + mul x9, x9, x2 adcs x13, x13, xzr - umulh x25, x12, x2 + mul x10, x10, x2 adc x14, x14, xzr - umulh x26, x13, x2 - smulh x28, x14, x2 - mul x9, x9, x2 - mul x10, x10, x2 - mul x11, x11, x2 + umulh x24, x11, x2 + and x28, x28, x2 + umulh x25, x12, x2 adds x10, x10, x22 + mul x11, x11, x2 + umulh x22, x13, x2 + neg x28, x28 mul x12, x12, x2 adcs x11, x11, x23 + umulh x23, x14, x2 mul x13, x13, x2 adcs x12, x12, x24 mul x14, x14, x2 adcs x13, x13, x25 - adcs x14, x14 ,x26 - adc x28, x28, xzr + adcs x14, x14, x22 + adc x28, x23, x28 adds x3, x3, x9 adcs x4, x4, x10 adcs x5, x5, x11 diff --git a/blst_src/build/win64/ct_is_square_mod_384-armv8.asm b/blst_src/build/win64/ct_is_square_mod_384-armv8.asm index 5d3ca14e..cb8682cd 100644 --- a/blst_src/build/win64/ct_is_square_mod_384-armv8.asm +++ b/blst_src/build/win64/ct_is_square_mod_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |ct_is_square_mod_384|[FUNC] ALIGN 32 |ct_is_square_mod_384| PROC - DCDU 3573752639 + hint #25 stp x29, x30, [sp,#-16*__SIZEOF_POINTER__]! add x29, sp, #0 stp x19, x20, [sp,#2*__SIZEOF_POINTER__] @@ -86,7 +86,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25, x26, [x29,#8*__SIZEOF_POINTER__] ldp x27, x28, [x29,#10*__SIZEOF_POINTER__] ldr x29, [sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP diff --git a/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm b/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm index 30c038cb..13aa9430 100644 --- a/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm +++ b/blst_src/build/win64/ctq_inverse_mod_384-x86_64.asm @@ -1,20 +1,22 @@ OPTION DOTNAME -EXTERN ct_inverse_mod_383$1:NEAR +ifdef __BLST_PORTABLE__ +EXTERN ct_inverse_mod_384$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS .text$ SEGMENT ALIGN(256) 'CODE' -PUBLIC ct_inverse_mod_383 +PUBLIC ct_inverse_mod_384 ALIGN 32 -ct_inverse_mod_383 PROC PUBLIC +ct_inverse_mod_384 PROC PUBLIC DB 243,15,30,250 mov QWORD PTR[8+rsp],rdi ;WIN64 prologue mov QWORD PTR[16+rsp],rsi mov r11,rsp -$L$SEH_begin_ct_inverse_mod_383:: +$L$SEH_begin_ct_inverse_mod_384:: mov rdi,rcx @@ -23,7 +25,7 @@ $L$SEH_begin_ct_inverse_mod_383:: mov rcx,r9 ifdef __BLST_PORTABLE__ test DWORD PTR[__blst_platform_cap],1 - jnz ct_inverse_mod_383$1 + jnz ct_inverse_mod_384$1 endif push rbp @@ -39,7 +41,7 @@ endif sub rsp,1112 -$L$SEH_body_ct_inverse_mod_383:: +$L$SEH_body_ct_inverse_mod_384:: lea rax,QWORD PTR[((88+511))+rsp] @@ -86,7 +88,7 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[96+rdi],rdx @@ -94,10 +96,10 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 - mov QWORD PTR[96+rdi],rdx + mov QWORD PTR[104+rdi],rdx xor rsi,256 @@ -110,19 +112,19 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov rax,QWORD PTR[96+rsi] - mov r11,QWORD PTR[144+rsi] + mov r11,QWORD PTR[152+rsi] mov rbx,rdx mov r10,rax imul QWORD PTR[56+rsp] @@ -139,6 +141,7 @@ $L$SEH_body_ct_inverse_mod_383:: mov QWORD PTR[72+rdi],r9 mov QWORD PTR[80+rdi],r9 mov QWORD PTR[88+rdi],r9 + mov QWORD PTR[96+rdi],r9 lea rsi,QWORD PTR[96+rsi] mov rax,r10 @@ -149,13 +152,14 @@ $L$SEH_body_ct_inverse_mod_383:: imul rcx add r8,rax adc r9,rdx - mov QWORD PTR[96+rdi],r8 - mov QWORD PTR[104+rdi],r9 - sar r9,63 + mov QWORD PTR[104+rdi],r8 mov QWORD PTR[112+rdi],r9 + sar r9,63 mov QWORD PTR[120+rdi],r9 mov QWORD PTR[128+rdi],r9 mov QWORD PTR[136+rdi],r9 + mov QWORD PTR[144+rdi],r9 + mov QWORD PTR[152+rdi],r9 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -166,14 +170,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -181,12 +185,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -197,14 +201,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -212,12 +216,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -228,14 +232,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -243,12 +247,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -259,14 +263,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -274,19 +278,17 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 - sar r13,63 - mov QWORD PTR[48+rdi],r13 - mov QWORD PTR[56+rdi],r13 - mov QWORD PTR[64+rdi],r13 - mov QWORD PTR[72+rdi],r13 - mov QWORD PTR[80+rdi],r13 - mov QWORD PTR[88+rdi],r13 + lea rdi,QWORD PTR[56+rdi] + call __smulq_384x63 + mov QWORD PTR[56+rdi],r14 + mov QWORD PTR[64+rdi],r14 + mov QWORD PTR[72+rdi],r14 + mov QWORD PTR[80+rdi],r14 + mov QWORD PTR[88+rdi],r14 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -297,14 +299,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -312,12 +314,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -328,14 +330,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -343,12 +345,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -359,14 +361,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -374,12 +376,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -390,14 +392,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -405,12 +407,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 call __ab_approximation_62 @@ -421,14 +423,14 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdi,256 xor rdi,rsi - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulq_383_n_shift_by_62 + call __smulq_384_n_shift_by_62 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -436,12 +438,12 @@ $L$SEH_body_ct_inverse_mod_383:: mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 mov edi,62 @@ -465,16 +467,16 @@ $L$SEH_body_ct_inverse_mod_383:: lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[96+rdi] - call __smulq_383x63 + call __smulq_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulq_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulq_768x63 xor rsi,256+8*12 - mov edi,22 + mov edi,24 mov r8,QWORD PTR[rsi] xor r9,r9 @@ -497,37 +499,77 @@ $L$SEH_body_ct_inverse_mod_383:: mov rdx,r12 mov rcx,r13 mov rdi,QWORD PTR[32+rsp] - call __smulq_767x63 + call __smulq_768x63 mov rsi,QWORD PTR[40+rsp] - mov rdx,rax - sar rax,63 + mov r13,rdx + sar r13,63 - mov r8,rax - mov r9,rax - mov r10,rax + mov r8,r13 + mov r9,r13 + mov r10,r13 + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,r13 + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,r13 + and r12,QWORD PTR[32+rsi] + and r13,QWORD PTR[40+rsi] + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rax,r13 + adc rdx,0 + + mov r13,rdx + neg rdx + or r13,rdx + sar rdx,63 + + mov r8,r13 + mov r9,r13 + mov r10,r13 and r8,QWORD PTR[rsi] and r9,QWORD PTR[8+rsi] - mov r11,rax + mov r11,r13 and r10,QWORD PTR[16+rsi] and r11,QWORD PTR[24+rsi] - mov r12,rax + mov r12,r13 and r12,QWORD PTR[32+rsi] - and rax,QWORD PTR[40+rsi] + and r13,QWORD PTR[40+rsi] + + xor r8,rdx + xor rsi,rsi + xor r9,rdx + sub rsi,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add r8,rsi + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 add r14,r8 adc r15,r9 adc rbx,r10 adc rbp,r11 adc rcx,r12 - adc rdx,rax + adc rax,r13 mov QWORD PTR[48+rdi],r14 mov QWORD PTR[56+rdi],r15 mov QWORD PTR[64+rdi],rbx mov QWORD PTR[72+rdi],rbp mov QWORD PTR[80+rdi],rcx - mov QWORD PTR[88+rdi],rdx + mov QWORD PTR[88+rdi],rax lea r8,QWORD PTR[1112+rsp] mov r15,QWORD PTR[r8] @@ -544,7 +586,7 @@ $L$SEH_body_ct_inverse_mod_383:: lea rsp,QWORD PTR[48+r8] -$L$SEH_epilogue_ct_inverse_mod_383:: +$L$SEH_epilogue_ct_inverse_mod_384:: mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue mov rsi,QWORD PTR[16+rsp] @@ -558,11 +600,11 @@ else DB 0F3h,0C3h endif -$L$SEH_end_ct_inverse_mod_383:: -ct_inverse_mod_383 ENDP +$L$SEH_end_ct_inverse_mod_384:: +ct_inverse_mod_384 ENDP ALIGN 32 -__smulq_767x63 PROC PRIVATE +__smulq_768x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -571,6 +613,7 @@ __smulq_767x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -579,7 +622,7 @@ __smulq_767x63 PROC PRIVATE mov QWORD PTR[8+rsp],rdi mov QWORD PTR[16+rsp],rsi - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] xor rbp,rdx add rbp,rax @@ -590,16 +633,20 @@ __smulq_767x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov QWORD PTR[rdi],rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -625,14 +672,14 @@ __smulq_767x63 PROC PRIVATE adc rdx,0 mov r13,rdx mov QWORD PTR[32+rdi],r12 - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r14,rdx mov QWORD PTR[40+rdi],r13 - mov QWORD PTR[48+rdi],rdx - sar rdx,63 - mov QWORD PTR[56+rdi],rdx + mov QWORD PTR[48+rdi],r14 + sar r14,63 + mov QWORD PTR[56+rdi],r14 mov rdx,rcx mov r8,QWORD PTR[rsi] @@ -735,39 +782,41 @@ __smulq_767x63 PROC PRIVATE mov rax,rdi adc rdx,0 mov rdi,rdx - mov rdx,QWORD PTR[8+rsp] - imul rax,rsi - mov rsi,QWORD PTR[16+rsp] + imul rsi + mov rsi,QWORD PTR[8+rsp] add rax,rdi + adc rdx,0 - add r8,QWORD PTR[rdx] - adc r9,QWORD PTR[8+rdx] - adc r10,QWORD PTR[16+rdx] - adc r11,QWORD PTR[24+rdx] - adc r12,QWORD PTR[32+rdx] - adc r13,QWORD PTR[40+rdx] - adc r14,QWORD PTR[48+rdx] - mov rdi,QWORD PTR[56+rdx] + add r8,QWORD PTR[rsi] + adc r9,QWORD PTR[8+rsi] + adc r10,QWORD PTR[16+rsi] + adc r11,QWORD PTR[24+rsi] + adc r12,QWORD PTR[32+rsi] + adc r13,QWORD PTR[40+rsi] + adc r14,QWORD PTR[48+rsi] + mov rdi,QWORD PTR[56+rsi] adc r15,rdi adc rbx,rdi adc rbp,rdi adc rcx,rdi adc rax,rdi + adc rdx,rdi - mov rdi,rdx + lea rdi,QWORD PTR[rsi] + mov rsi,QWORD PTR[16+rsp] - mov QWORD PTR[rdx],r8 - mov QWORD PTR[8+rdx],r9 - mov QWORD PTR[16+rdx],r10 - mov QWORD PTR[24+rdx],r11 - mov QWORD PTR[32+rdx],r12 - mov QWORD PTR[40+rdx],r13 - mov QWORD PTR[48+rdx],r14 - mov QWORD PTR[56+rdx],r15 - mov QWORD PTR[64+rdx],rbx - mov QWORD PTR[72+rdx],rbp - mov QWORD PTR[80+rdx],rcx - mov QWORD PTR[88+rdx],rax + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rax ifdef __SGX_LVI_HARDENING__ @@ -778,10 +827,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_767x63 ENDP +__smulq_768x63 ENDP ALIGN 32 -__smulq_383x63 PROC PRIVATE +__smulq_384x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -790,6 +839,7 @@ __smulq_383x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -805,16 +855,20 @@ __smulq_383x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -836,10 +890,11 @@ __smulq_383x63 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rax,rbp + mul rbp add r13,rax + adc r14,rdx - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] mov rdx,rcx mov QWORD PTR[rdi],r8 @@ -847,13 +902,15 @@ __smulq_383x63 PROC PRIVATE mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],r13 + mov r15,r13 + mov rbx,r14 mov r8,QWORD PTR[rsi] mov r9,QWORD PTR[8+rsi] mov r10,QWORD PTR[16+rsi] mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rbp,rdx sar rdx,63 @@ -869,16 +926,20 @@ __smulq_383x63 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + xor r14,rdx add rax,r8 adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -900,17 +961,19 @@ __smulq_383x63 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rax,rbp + mul rbp add r13,rax + adc r14,rdx - lea rsi,QWORD PTR[((-48))+rsi] + lea rsi,QWORD PTR[((-56))+rsi] add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc r13,QWORD PTR[40+rdi] + adc r13,r15 + adc r14,rbx mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 @@ -918,6 +981,7 @@ __smulq_383x63 PROC PRIVATE mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 ifdef __SGX_LVI_HARDENING__ @@ -928,10 +992,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_383x63 ENDP +__smulq_384x63 ENDP ALIGN 32 -__smulq_383_n_shift_by_62 PROC PRIVATE +__smulq_384_n_shift_by_62 PROC PRIVATE DB 243,15,30,250 mov rbx,rdx @@ -956,6 +1020,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + mov r14,rdx add rax,r8 adc r9,0 adc r10,0 @@ -966,6 +1031,8 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mul rbp mov r8,rax mov rax,r9 + and r14,rbp + neg r14 mov r9,rdx mul rbp add r9,rax @@ -987,12 +1054,11 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r14,rdx lea rsi,QWORD PTR[48+rsi] - mov r14,rdx mov rdx,rcx mov QWORD PTR[rdi],r8 @@ -1022,6 +1088,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE xor r11,rdx xor r12,rdx xor r13,rdx + mov r15,rdx add rax,r8 adc r9,0 adc r10,0 @@ -1032,6 +1099,8 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mul rbp mov r8,rax mov rax,r9 + and r15,rbp + neg r15 mov r9,rdx mul rbp add r9,rax @@ -1053,11 +1122,12 @@ __smulq_383_n_shift_by_62 PROC PRIVATE mov rax,r13 adc rdx,0 mov r13,rdx - imul rbp + mul rbp add r13,rax - adc rdx,0 + adc r15,rdx lea rsi,QWORD PTR[((-48))+rsi] + mov rdx,rbx add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] @@ -1065,8 +1135,7 @@ __smulq_383_n_shift_by_62 PROC PRIVATE adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] adc r13,QWORD PTR[40+rdi] - adc r14,rdx - mov rdx,rbx + adc r14,r15 shrd r8,r9,62 shrd r9,r10,62 @@ -1113,7 +1182,7 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulq_383_n_shift_by_62 ENDP +__smulq_384_n_shift_by_62 ENDP ALIGN 32 __ab_approximation_62 PROC PRIVATE @@ -1242,29 +1311,29 @@ __inner_loop_62 ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 - DD imagerel $L$SEH_begin_ct_inverse_mod_383 - DD imagerel $L$SEH_body_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_prologue + DD imagerel $L$SEH_begin_ct_inverse_mod_384 + DD imagerel $L$SEH_body_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_prologue - DD imagerel $L$SEH_body_ct_inverse_mod_383 - DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_body + DD imagerel $L$SEH_body_ct_inverse_mod_384 + DD imagerel $L$SEH_epilogue_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_body - DD imagerel $L$SEH_epilogue_ct_inverse_mod_383 - DD imagerel $L$SEH_end_ct_inverse_mod_383 - DD imagerel $L$SEH_info_ct_inverse_mod_383_epilogue + DD imagerel $L$SEH_epilogue_ct_inverse_mod_384 + DD imagerel $L$SEH_end_ct_inverse_mod_384 + DD imagerel $L$SEH_info_ct_inverse_mod_384_epilogue .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 -$L$SEH_info_ct_inverse_mod_383_prologue:: +$L$SEH_info_ct_inverse_mod_384_prologue:: DB 1,0,5,00bh DB 0,074h,1,0 DB 0,064h,2,0 DB 0,0b3h DB 0,0 DD 0,0 -$L$SEH_info_ct_inverse_mod_383_body:: +$L$SEH_info_ct_inverse_mod_384_body:: DB 1,0,18,0 DB 000h,0f4h,08bh,000h DB 000h,0e4h,08ch,000h @@ -1277,7 +1346,7 @@ DB 000h,064h,093h,000h DB 000h,001h,091h,000h DB 000h,000h,000h,000h DB 000h,000h,000h,000h -$L$SEH_info_ct_inverse_mod_383_epilogue:: +$L$SEH_info_ct_inverse_mod_384_epilogue:: DB 1,0,4,0 DB 000h,074h,001h,000h DB 000h,064h,002h,000h diff --git a/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm b/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm index cfc2141c..90d23389 100644 --- a/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm +++ b/blst_src/build/win64/ctx_inverse_mod_384-x86_64.asm @@ -1,24 +1,26 @@ OPTION DOTNAME -PUBLIC ct_inverse_mod_383$1 +ifdef __BLST_PORTABLE__ +PUBLIC ct_inverse_mod_384$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' -PUBLIC ctx_inverse_mod_383 +PUBLIC ctx_inverse_mod_384 ALIGN 32 -ctx_inverse_mod_383 PROC PUBLIC +ctx_inverse_mod_384 PROC PUBLIC DB 243,15,30,250 mov QWORD PTR[8+rsp],rdi ;WIN64 prologue mov QWORD PTR[16+rsp],rsi mov r11,rsp -$L$SEH_begin_ctx_inverse_mod_383:: +$L$SEH_begin_ctx_inverse_mod_384:: mov rdi,rcx mov rsi,rdx mov rdx,r8 mov rcx,r9 -ct_inverse_mod_383$1:: +ct_inverse_mod_384$1:: push rbp push rbx @@ -33,7 +35,7 @@ ct_inverse_mod_383$1:: sub rsp,1112 -$L$SEH_body_ctx_inverse_mod_383:: +$L$SEH_body_ctx_inverse_mod_384:: lea rax,QWORD PTR[((88+511))+rsp] @@ -83,7 +85,7 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[96+rdi],rdx @@ -91,10 +93,10 @@ endif mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 - mov QWORD PTR[96+rdi],rdx + mov QWORD PTR[104+rdi],rdx xor rsi,256 @@ -107,19 +109,19 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov rax,QWORD PTR[96+rsi] - mov r11,QWORD PTR[144+rsi] + mov r11,QWORD PTR[152+rsi] mov rbx,rdx mov r10,rax imul QWORD PTR[56+rsp] @@ -136,6 +138,7 @@ endif mov QWORD PTR[72+rdi],r9 mov QWORD PTR[80+rdi],r9 mov QWORD PTR[88+rdi],r9 + mov QWORD PTR[96+rdi],r9 lea rsi,QWORD PTR[96+rsi] mov rax,r10 @@ -146,13 +149,14 @@ endif imul rcx add r8,rax adc r9,rdx - mov QWORD PTR[96+rdi],r8 - mov QWORD PTR[104+rdi],r9 - sar r9,63 + mov QWORD PTR[104+rdi],r8 mov QWORD PTR[112+rdi],r9 + sar r9,63 mov QWORD PTR[120+rdi],r9 mov QWORD PTR[128+rdi],r9 mov QWORD PTR[136+rdi],r9 + mov QWORD PTR[144+rdi],r9 + mov QWORD PTR[152+rdi],r9 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -163,14 +167,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -178,12 +182,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -194,14 +198,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -209,12 +213,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -225,14 +229,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -240,12 +244,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -256,14 +260,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -271,12 +275,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -287,14 +291,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -302,12 +306,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -318,14 +322,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -333,12 +337,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -349,14 +353,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -364,12 +368,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -380,14 +384,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -395,12 +399,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -411,14 +415,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -426,12 +430,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -442,14 +446,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -457,19 +461,17 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 - sar r13,63 - mov QWORD PTR[48+rdi],r13 - mov QWORD PTR[56+rdi],r13 - mov QWORD PTR[64+rdi],r13 - mov QWORD PTR[72+rdi],r13 - mov QWORD PTR[80+rdi],r13 - mov QWORD PTR[88+rdi],r13 + lea rdi,QWORD PTR[56+rdi] + call __smulx_384x63 + mov QWORD PTR[56+rdi],r14 + mov QWORD PTR[64+rdi],r14 + mov QWORD PTR[72+rdi],r14 + mov QWORD PTR[80+rdi],r14 + mov QWORD PTR[88+rdi],r14 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -480,14 +482,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -495,12 +497,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -511,14 +513,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -526,12 +528,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -542,14 +544,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -557,12 +559,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -573,14 +575,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -588,12 +590,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -604,14 +606,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -619,12 +621,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -635,14 +637,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -650,12 +652,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -666,14 +668,14 @@ endif mov rdi,256 xor rdi,rsi - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[56+rsp],rdx mov QWORD PTR[64+rsp],rcx mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] lea rdi,QWORD PTR[48+rdi] - call __smulx_383_n_shift_by_31 + call __smulx_384_n_shift_by_31 mov QWORD PTR[72+rsp],rdx mov QWORD PTR[80+rsp],rcx @@ -681,12 +683,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -712,12 +714,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -743,12 +745,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -774,12 +776,12 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 mov edi,31 call __ab_approximation_31 @@ -805,21 +807,21 @@ endif mov rcx,QWORD PTR[64+rsp] lea rsi,QWORD PTR[96+rsi] lea rdi,QWORD PTR[48+rdi] - call __smulx_383x63 + call __smulx_384x63 mov rdx,QWORD PTR[72+rsp] mov rcx,QWORD PTR[80+rsp] - lea rdi,QWORD PTR[48+rdi] - call __smulx_767x63 + lea rdi,QWORD PTR[56+rdi] + call __smulx_768x63 xor rsi,256+8*12 - mov edi,53 + mov edi,55 mov r8,QWORD PTR[rsi] mov r10,QWORD PTR[48+rsi] - call __tail_loop_53 + call __tail_loop_55 @@ -836,40 +838,80 @@ endif mov rdx,r12 mov rcx,r13 mov rdi,QWORD PTR[32+rsp] - call __smulx_767x63 + call __smulx_768x63 mov rsi,QWORD PTR[40+rsp] - mov rdx,rax - sar rax,63 + mov r13,rdx + sar r13,63 - mov r8,rax - mov r9,rax - mov r10,rax + mov r8,r13 + mov r9,r13 + mov r10,r13 ifdef __SGX_LVI_HARDENING__ lfence endif and r8,QWORD PTR[rsi] and r9,QWORD PTR[8+rsi] - mov r11,rax + mov r11,r13 and r10,QWORD PTR[16+rsi] and r11,QWORD PTR[24+rsi] - mov r12,rax + mov r12,r13 and r12,QWORD PTR[32+rsi] - and rax,QWORD PTR[40+rsi] + and r13,QWORD PTR[40+rsi] add r14,r8 adc r15,r9 adc rbx,r10 adc rbp,r11 adc rcx,r12 - adc rdx,rax + adc rax,r13 + adc rdx,0 + + mov r13,rdx + neg rdx + or r13,rdx + sar rdx,63 + + mov r8,r13 + mov r9,r13 + mov r10,r13 + and r8,QWORD PTR[rsi] + and r9,QWORD PTR[8+rsi] + mov r11,r13 + and r10,QWORD PTR[16+rsi] + and r11,QWORD PTR[24+rsi] + mov r12,r13 + and r12,QWORD PTR[32+rsi] + and r13,QWORD PTR[40+rsi] + + xor r8,rdx + xor rsi,rsi + xor r9,rdx + sub rsi,rdx + xor r10,rdx + xor r11,rdx + xor r12,rdx + xor r13,rdx + add r8,rsi + adc r9,0 + adc r10,0 + adc r11,0 + adc r12,0 + adc r13,0 + + add r14,r8 + adc r15,r9 + adc rbx,r10 + adc rbp,r11 + adc rcx,r12 + adc rax,r13 mov QWORD PTR[48+rdi],r14 mov QWORD PTR[56+rdi],r15 mov QWORD PTR[64+rdi],rbx mov QWORD PTR[72+rdi],rbp mov QWORD PTR[80+rdi],rcx - mov QWORD PTR[88+rdi],rdx + mov QWORD PTR[88+rdi],rax lea r8,QWORD PTR[1112+rsp] mov r15,QWORD PTR[r8] @@ -886,7 +928,7 @@ endif lea rsp,QWORD PTR[48+r8] -$L$SEH_epilogue_ctx_inverse_mod_383:: +$L$SEH_epilogue_ctx_inverse_mod_384:: mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue mov rsi,QWORD PTR[16+rsp] @@ -900,11 +942,11 @@ else DB 0F3h,0C3h endif -$L$SEH_end_ctx_inverse_mod_383:: -ctx_inverse_mod_383 ENDP +$L$SEH_end_ctx_inverse_mod_384:: +ctx_inverse_mod_384 ENDP ALIGN 32 -__smulx_767x63 PROC PRIVATE +__smulx_768x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[rsi] @@ -913,6 +955,7 @@ __smulx_767x63 PROC PRIVATE mov r11,QWORD PTR[24+rsi] mov r12,QWORD PTR[32+rsi] mov r13,QWORD PTR[40+rsi] + mov r14,QWORD PTR[48+rsi] mov rax,rdx sar rax,63 @@ -921,7 +964,7 @@ __smulx_767x63 PROC PRIVATE mov QWORD PTR[8+rsp],rdi mov QWORD PTR[16+rsp],rsi - lea rsi,QWORD PTR[48+rsi] + lea rsi,QWORD PTR[56+rsi] xor rdx,rax add rdx,rbp @@ -931,37 +974,41 @@ __smulx_767x63 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax + xor r14,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx rax,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,rax + mulx rax,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc rdx,0 + adc r12,rax + mulx rax,r13,r13 + adc r13,rbp + adc r14,rax mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax - mov QWORD PTR[48+rdi],rdx - sar rdx,63 - mov QWORD PTR[56+rdi],rdx + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + sar r14,63 + mov QWORD PTR[56+rdi],r14 mov rdx,rcx mov rax,rcx @@ -996,7 +1043,7 @@ __smulx_767x63 PROC PRIVATE xor rbx,rax xor rbp,rax xor rcx,rax - xor rdi,rax + xor rax,rdi add r8,rsi adc r9,0 adc r10,0 @@ -1008,62 +1055,64 @@ __smulx_767x63 PROC PRIVATE adc rbx,0 adc rbp,0 adc rcx,0 - adc rdi,0 - - mulx rax,r8,r8 - mulx rsi,r9,r9 - add r9,rax - mulx rax,r10,r10 - adc r10,rsi - mulx rsi,r11,r11 - adc r11,rax - mulx rax,r12,r12 - adc r12,rsi - mulx rsi,r13,r13 - adc r13,rax - mulx rax,r14,r14 - adc r14,rsi - mulx rsi,r15,r15 - adc r15,rax - mulx rax,rbx,rbx + adc rax,0 + + mulx rsi,r8,r8 + mulx rdi,r9,r9 + add r9,rsi + mulx rsi,r10,r10 + adc r10,rdi + mulx rdi,r11,r11 + adc r11,rsi + mulx rsi,r12,r12 + adc r12,rdi + mulx rdi,r13,r13 + adc r13,rsi + mulx rsi,r14,r14 + adc r14,rdi + mulx rdi,r15,r15 + adc r15,rsi + mulx rsi,rbx,rbx + adc rbx,rdi + mulx rdi,rbp,rbp + adc rbp,rsi + mulx rsi,rcx,rcx + adc rcx,rdi + mov rdi,QWORD PTR[8+rsp] + adc rsi,0 + imul rdx + add rax,rsi + adc rdx,0 + + add r8,QWORD PTR[rdi] + adc r9,QWORD PTR[8+rdi] + adc r10,QWORD PTR[16+rdi] + adc r11,QWORD PTR[24+rdi] + adc r12,QWORD PTR[32+rdi] + adc r13,QWORD PTR[40+rdi] + adc r14,QWORD PTR[48+rdi] + mov rsi,QWORD PTR[56+rdi] + adc r15,rsi adc rbx,rsi - mulx rsi,rbp,rbp - adc rbp,rax - mulx rax,rcx,rcx + adc rbp,rsi adc rcx,rsi - mulx rsi,rdi,rdi - mov rdx,QWORD PTR[8+rsp] + adc rax,rsi + adc rdx,rsi + mov rsi,QWORD PTR[16+rsp] - adc rax,rdi - - add r8,QWORD PTR[rdx] - adc r9,QWORD PTR[8+rdx] - adc r10,QWORD PTR[16+rdx] - adc r11,QWORD PTR[24+rdx] - adc r12,QWORD PTR[32+rdx] - adc r13,QWORD PTR[40+rdx] - adc r14,QWORD PTR[48+rdx] - mov rdi,QWORD PTR[56+rdx] - adc r15,rdi - adc rbx,rdi - adc rbp,rdi - adc rcx,rdi - adc rax,rdi - - mov rdi,rdx - - mov QWORD PTR[rdx],r8 - mov QWORD PTR[8+rdx],r9 - mov QWORD PTR[16+rdx],r10 - mov QWORD PTR[24+rdx],r11 - mov QWORD PTR[32+rdx],r12 - mov QWORD PTR[40+rdx],r13 - mov QWORD PTR[48+rdx],r14 - mov QWORD PTR[56+rdx],r15 - mov QWORD PTR[64+rdx],rbx - mov QWORD PTR[72+rdx],rbp - mov QWORD PTR[80+rdx],rcx - mov QWORD PTR[88+rdx],rax + + mov QWORD PTR[rdi],r8 + mov QWORD PTR[8+rdi],r9 + mov QWORD PTR[16+rdi],r10 + mov QWORD PTR[24+rdi],r11 + mov QWORD PTR[32+rdi],r12 + mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 + mov QWORD PTR[56+rdi],r15 + mov QWORD PTR[64+rdi],rbx + mov QWORD PTR[72+rdi],rbp + mov QWORD PTR[80+rdi],rcx + mov QWORD PTR[88+rdi],rax ifdef __SGX_LVI_HARDENING__ @@ -1074,10 +1123,10 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_767x63 ENDP +__smulx_768x63 ENDP ALIGN 32 -__smulx_383x63 PROC PRIVATE +__smulx_384x63 PROC PRIVATE DB 243,15,30,250 mov r8,QWORD PTR[((0+0))+rsi] @@ -1086,6 +1135,7 @@ __smulx_383x63 PROC PRIVATE mov r11,QWORD PTR[((0+24))+rsi] mov r12,QWORD PTR[((0+32))+rsi] mov r13,QWORD PTR[((0+40))+rsi] + mov r14,QWORD PTR[((0+48))+rsi] mov rbp,rdx sar rbp,63 @@ -1101,12 +1151,17 @@ __smulx_383x63 PROC PRIVATE xor r11,rbp xor r12,rbp xor r13,rbp + xor r14,rbp add r8,rax adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 mulx rax,r9,r9 @@ -1120,19 +1175,22 @@ __smulx_383x63 PROC PRIVATE mulx rax,r13,r13 mov rdx,rcx adc r13,rbp + adc r14,rax mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],r13 - mov r8,QWORD PTR[((48+0))+rsi] - mov r9,QWORD PTR[((48+8))+rsi] - mov r10,QWORD PTR[((48+16))+rsi] - mov r11,QWORD PTR[((48+24))+rsi] - mov r12,QWORD PTR[((48+32))+rsi] - mov r13,QWORD PTR[((48+40))+rsi] + mov r15,r13 + mov rbx,r14 + mov r8,QWORD PTR[((56+0))+rsi] + mov r9,QWORD PTR[((56+8))+rsi] + mov r10,QWORD PTR[((56+16))+rsi] + mov r11,QWORD PTR[((56+24))+rsi] + mov r12,QWORD PTR[((56+32))+rsi] + mov r13,QWORD PTR[((56+40))+rsi] + mov r14,QWORD PTR[((56+48))+rsi] mov rbp,rdx sar rbp,63 @@ -1148,12 +1206,17 @@ __smulx_383x63 PROC PRIVATE xor r11,rbp xor r12,rbp xor r13,rbp + xor r14,rbp add r8,rax adc r9,0 adc r10,0 adc r11,0 adc r12,0 adc r13,0 + adc r14,0 + + and r14,rdx + neg r14 mulx rbp,r8,r8 mulx rax,r9,r9 @@ -1166,13 +1229,15 @@ __smulx_383x63 PROC PRIVATE adc r12,rax mulx rax,r13,r13 adc r13,rbp + adc r14,rax add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc r13,QWORD PTR[40+rdi] + adc r13,r15 + adc r14,rbx mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 @@ -1180,6 +1245,7 @@ __smulx_383x63 PROC PRIVATE mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 mov QWORD PTR[40+rdi],r13 + mov QWORD PTR[48+rdi],r14 ifdef __SGX_LVI_HARDENING__ @@ -1190,14 +1256,13 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_383x63 ENDP +__smulx_384x63 ENDP ALIGN 32 -__smulx_383_n_shift_by_31 PROC PRIVATE +__smulx_384_n_shift_by_31 PROC PRIVATE DB 243,15,30,250 mov rbx,rdx - xor r14,r14 mov r8,QWORD PTR[((0+0))+rsi] mov r9,QWORD PTR[((0+8))+rsi] mov r10,QWORD PTR[((0+16))+rsi] @@ -1218,27 +1283,29 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + + and rax,rdx + neg rax mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx r14,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,r14 + mulx r14,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc r14,rdx + adc r12,r14 + mulx r14,r13,r13 + adc r13,rbp + adc r14,rax mov rdx,rcx @@ -1247,7 +1314,8 @@ __smulx_383_n_shift_by_31 PROC PRIVATE mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax + mov QWORD PTR[40+rdi],r13 + mov r15,r14 mov r8,QWORD PTR[((48+0))+rsi] mov r9,QWORD PTR[((48+8))+rsi] mov r10,QWORD PTR[((48+16))+rsi] @@ -1268,43 +1336,45 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,rax xor r11,rax xor r12,rax - xor rax,r13 + xor r13,rax add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 + + and rax,rdx + neg rax mulx rbp,r8,r8 - mulx r13,r9,r9 + mulx r14,r9,r9 add r9,rbp mulx rbp,r10,r10 - adc r10,r13 - mulx r13,r11,r11 + adc r10,r14 + mulx r14,r11,r11 adc r11,rbp mulx rbp,r12,r12 - adc r12,r13 - adc rbp,0 - imul rdx - add rax,rbp - adc rdx,0 + adc r12,r14 + mulx r14,r13,r13 + adc r13,rbp + adc r14,rax add r8,QWORD PTR[rdi] adc r9,QWORD PTR[8+rdi] adc r10,QWORD PTR[16+rdi] adc r11,QWORD PTR[24+rdi] adc r12,QWORD PTR[32+rdi] - adc rax,QWORD PTR[40+rdi] - adc r14,rdx + adc r13,QWORD PTR[40+rdi] + adc r14,r15 mov rdx,rbx shrd r8,r9,31 shrd r9,r10,31 shrd r10,r11,31 shrd r11,r12,31 - shrd r12,rax,31 - shrd rax,r14,31 + shrd r12,r13,31 + shrd r13,r14,31 sar r14,63 xor rbp,rbp @@ -1315,20 +1385,20 @@ __smulx_383_n_shift_by_31 PROC PRIVATE xor r10,r14 xor r11,r14 xor r12,r14 - xor rax,r14 + xor r13,r14 add r8,rbp adc r9,0 adc r10,0 adc r11,0 adc r12,0 - adc rax,0 + adc r13,0 mov QWORD PTR[rdi],r8 mov QWORD PTR[8+rdi],r9 mov QWORD PTR[16+rdi],r10 mov QWORD PTR[24+rdi],r11 mov QWORD PTR[32+rdi],r12 - mov QWORD PTR[40+rdi],rax + mov QWORD PTR[40+rdi],r13 xor rdx,r14 xor rcx,r14 @@ -1344,7 +1414,7 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__smulx_383_n_shift_by_31 ENDP +__smulx_384_n_shift_by_31 ENDP ALIGN 32 __smulx_191_n_shift_by_31 PROC PRIVATE @@ -1583,7 +1653,7 @@ __inner_loop_31 ENDP ALIGN 32 -__tail_loop_53 PROC PRIVATE +__tail_loop_55 PROC PRIVATE DB 243,15,30,250 mov rdx,1 @@ -1591,7 +1661,7 @@ __tail_loop_53 PROC PRIVATE xor r12,r12 mov r13,1 -$L$oop_53:: +$L$oop_55:: xor rax,rax test r8,1 mov rbx,r10 @@ -1618,7 +1688,7 @@ $L$oop_53:: sub rdx,rax sub rcx,rbx sub edi,1 - jnz $L$oop_53 + jnz $L$oop_55 ifdef __SGX_LVI_HARDENING__ @@ -1629,33 +1699,33 @@ ifdef __SGX_LVI_HARDENING__ else DB 0F3h,0C3h endif -__tail_loop_53 ENDP +__tail_loop_55 ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 - DD imagerel $L$SEH_begin_ctx_inverse_mod_383 - DD imagerel $L$SEH_body_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_prologue + DD imagerel $L$SEH_begin_ctx_inverse_mod_384 + DD imagerel $L$SEH_body_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_prologue - DD imagerel $L$SEH_body_ctx_inverse_mod_383 - DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_body + DD imagerel $L$SEH_body_ctx_inverse_mod_384 + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_body - DD imagerel $L$SEH_epilogue_ctx_inverse_mod_383 - DD imagerel $L$SEH_end_ctx_inverse_mod_383 - DD imagerel $L$SEH_info_ctx_inverse_mod_383_epilogue + DD imagerel $L$SEH_epilogue_ctx_inverse_mod_384 + DD imagerel $L$SEH_end_ctx_inverse_mod_384 + DD imagerel $L$SEH_info_ctx_inverse_mod_384_epilogue .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 -$L$SEH_info_ctx_inverse_mod_383_prologue:: +$L$SEH_info_ctx_inverse_mod_384_prologue:: DB 1,0,5,00bh DB 0,074h,1,0 DB 0,064h,2,0 DB 0,0b3h DB 0,0 DD 0,0 -$L$SEH_info_ctx_inverse_mod_383_body:: +$L$SEH_info_ctx_inverse_mod_384_body:: DB 1,0,18,0 DB 000h,0f4h,08bh,000h DB 000h,0e4h,08ch,000h @@ -1668,7 +1738,7 @@ DB 000h,064h,093h,000h DB 000h,001h,091h,000h DB 000h,000h,000h,000h DB 000h,000h,000h,000h -$L$SEH_info_ctx_inverse_mod_383_epilogue:: +$L$SEH_info_ctx_inverse_mod_384_epilogue:: DB 1,0,4,0 DB 000h,074h,001h,000h DB 000h,064h,002h,000h diff --git a/blst_src/build/win64/div3w-armv8.asm b/blst_src/build/win64/div3w-armv8.asm index 25659de2..27c37e2b 100644 --- a/blst_src/build/win64/div3w-armv8.asm +++ b/blst_src/build/win64/div3w-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |div_3_limbs|[FUNC] ALIGN 32 |div_3_limbs| PROC + hint #34 ldp x4,x5,[x0] eor x0,x0,x0 mov x3,#64 @@ -41,6 +42,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |quot_rem_128|[FUNC] ALIGN 32 |quot_rem_128| PROC + hint #34 ldp x3,x4,[x1] mul x5,x3,x2 @@ -78,6 +80,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |quot_rem_64|[FUNC] ALIGN 32 |quot_rem_64| PROC + hint #34 ldr x3,[x1] ldr x8,[x0] diff --git a/blst_src/build/win64/mul_mont_256-armv8.asm b/blst_src/build/win64/mul_mont_256-armv8.asm index 9bb3987c..54387a09 100644 --- a/blst_src/build/win64/mul_mont_256-armv8.asm +++ b/blst_src/build/win64/mul_mont_256-armv8.asm @@ -7,6 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_sparse_256|[FUNC] ALIGN 32 |mul_mont_sparse_256| PROC + hint #34 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -198,7 +199,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_sparse_256|[FUNC] ALIGN 32 |sqr_mont_sparse_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-6*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -299,7 +300,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldr x29,[sp],#6*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -307,7 +308,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |from_mont_256|[FUNC] ALIGN 32 |from_mont_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -332,7 +333,7 @@ __SIZEOF_POINTER__ SETA 64/8 stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -341,7 +342,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |redc_mont_256|[FUNC] ALIGN 32 |redc_mont_256| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -376,7 +377,7 @@ __SIZEOF_POINTER__ SETA 64/8 stp x12,x13,[x0,#16] ldr x29,[sp],#2*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP diff --git a/blst_src/build/win64/mul_mont_384-armv8.asm b/blst_src/build/win64/mul_mont_384-armv8.asm index d1213a90..26032557 100644 --- a/blst_src/build/win64/mul_mont_384-armv8.asm +++ b/blst_src/build/win64/mul_mont_384-armv8.asm @@ -7,7 +7,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |add_mod_384x384|[FUNC] ALIGN 32 |add_mod_384x384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -25,7 +25,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -89,7 +89,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sub_mod_384x384|[FUNC] ALIGN 32 |sub_mod_384x384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-8*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -107,7 +107,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] ldr x29,[sp],#8*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -243,7 +243,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_384x|[FUNC] ALIGN 32 |mul_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -316,7 +316,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -325,7 +325,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_384x|[FUNC] ALIGN 32 |sqr_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -401,7 +401,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -410,7 +410,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_mont_384|[FUNC] ALIGN 32 |mul_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -442,7 +442,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -823,7 +823,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_mont_384|[FUNC] ALIGN 32 |sqr_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -860,7 +860,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -869,7 +869,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_n_mul_mont_383|[FUNC] ALIGN 32 |sqr_n_mul_mont_383| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -925,7 +925,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1046,7 +1046,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_384|[FUNC] ALIGN 32 |sqr_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1068,7 +1068,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1077,7 +1077,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |redc_mont_384|[FUNC] ALIGN 32 |redc_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1101,7 +1101,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1110,7 +1110,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |from_mont_384|[FUNC] ALIGN 32 |from_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1151,7 +1151,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1369,7 +1369,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_384|[FUNC] ALIGN 32 |mul_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1387,7 +1387,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1573,7 +1573,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |mul_382x|[FUNC] ALIGN 32 |mul_382x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1655,7 +1655,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -1664,7 +1664,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sqr_382x|[FUNC] ALIGN 32 |sqr_382x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -1755,471 +1755,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 - ret - ENDP - - - - EXPORT |sqr_mont_382x|[FUNC] - ALIGN 32 -|sqr_mont_382x| PROC - DCDU 3573752639 - stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! - add x29,sp,#0 - stp x19,x20,[sp,#2*__SIZEOF_POINTER__] - stp x21,x22,[sp,#4*__SIZEOF_POINTER__] - stp x23,x24,[sp,#6*__SIZEOF_POINTER__] - stp x25,x26,[sp,#8*__SIZEOF_POINTER__] - stp x27,x28,[sp,#10*__SIZEOF_POINTER__] - stp x3,x0,[sp,#12*__SIZEOF_POINTER__] - sub sp,sp,#112 - mov x4,x3 - - ldp x11,x12,[x1] - ldp x13,x14,[x1,#16] - ldp x15,x16,[x1,#32] - - ldp x17,x20,[x1,#48] - ldp x21,x22,[x1,#64] - ldp x23,x24,[x1,#80] - - adds x5,x11,x17 - adcs x6,x12,x20 - adcs x7,x13,x21 - adcs x8,x14,x22 - adcs x9,x15,x23 - adc x10,x16,x24 - - subs x19,x11,x17 - sbcs x20,x12,x20 - sbcs x21,x13,x21 - sbcs x22,x14,x22 - sbcs x23,x15,x23 - sbcs x24,x16,x24 - sbc x25,xzr,xzr - - stp x5,x6,[sp] - stp x7,x8,[sp,#16] - stp x9,x10,[sp,#32] - stp x19,x20,[sp,#48] - stp x21,x22,[sp,#64] - stp x23,x24,[sp,#80] - str x25,[sp,#96] - - ldp x5,x6,[x2] - ldp x7,x8,[x2,#16] - ldp x9,x10,[x2,#32] - - add x2,x1,#48 - bl __mul_mont_383_nonred - - adds x19,x11,x11 - adcs x20,x12,x12 - adcs x21,x13,x13 - adcs x22,x14,x14 - adcs x23,x15,x15 - adc x24,x16,x16 - - stp x19,x20,[x2,#48] - stp x21,x22,[x2,#64] - stp x23,x24,[x2,#80] - - ldp x11,x12,[sp] - ldr x17,[sp,#48] - ldp x13,x14,[sp,#16] - ldp x15,x16,[sp,#32] - - add x2,sp,#48 - bl __mul_mont_383_nonred - ldr x30,[x29,#__SIZEOF_POINTER__] - - ldr x25,[sp,#96] - ldp x19,x20,[sp] - ldp x21,x22,[sp,#16] - ldp x23,x24,[sp,#32] - - and x19,x19,x25 - and x20,x20,x25 - and x21,x21,x25 - and x22,x22,x25 - and x23,x23,x25 - and x24,x24,x25 - - subs x11,x11,x19 - sbcs x12,x12,x20 - sbcs x13,x13,x21 - sbcs x14,x14,x22 - sbcs x15,x15,x23 - sbcs x16,x16,x24 - sbc x25,xzr,xzr - - and x19,x5,x25 - and x20,x6,x25 - and x21,x7,x25 - and x22,x8,x25 - and x23,x9,x25 - and x24,x10,x25 - - adds x11,x11,x19 - adcs x12,x12,x20 - adcs x13,x13,x21 - adcs x14,x14,x22 - adcs x15,x15,x23 - adc x16,x16,x24 - - stp x11,x12,[x2] - stp x13,x14,[x2,#16] - stp x15,x16,[x2,#32] - - add sp,sp,#112 - ldp x19,x20,[x29,#2*__SIZEOF_POINTER__] - ldp x21,x22,[x29,#4*__SIZEOF_POINTER__] - ldp x23,x24,[x29,#6*__SIZEOF_POINTER__] - ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] - ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] - ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 - ret - ENDP - - - ALIGN 32 -|__mul_mont_383_nonred| PROC - mul x19,x11,x17 - mul x20,x12,x17 - mul x21,x13,x17 - mul x22,x14,x17 - mul x23,x15,x17 - mul x24,x16,x17 - mul x4,x4,x19 - - umulh x26,x11,x17 - umulh x27,x12,x17 - umulh x28,x13,x17 - umulh x0,x14,x17 - umulh x1,x15,x17 - umulh x3,x16,x17 - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,xzr, x3 - mul x3,x10,x4 - ldr x17,[x2,8*1] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*2] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*3] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*4] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - ldr x17,[x2,8*5] - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - - ldr x4,[x29,#12*__SIZEOF_POINTER__] - adds x19,x20,x26 - mul x26,x11,x17 - adcs x20,x21,x27 - mul x27,x12,x17 - adcs x21,x22,x28 - mul x28,x13,x17 - adcs x22,x23,x0 - mul x0,x14,x17 - adcs x23,x24,x1 - mul x1,x15,x17 - adcs x24,x25,x3 - mul x3,x16,x17 - adc x25,xzr,xzr - - adds x19,x19,x26 - umulh x26,x11,x17 - adcs x20,x20,x27 - umulh x27,x12,x17 - adcs x21,x21,x28 - mul x4,x4,x19 - umulh x28,x13,x17 - adcs x22,x22,x0 - umulh x0,x14,x17 - adcs x23,x23,x1 - umulh x1,x15,x17 - adcs x24,x24,x3 - umulh x3,x16,x17 - adc x25,x25,xzr - - adds x20,x20,x26 - mul x26,x5,x4 - adcs x21,x21,x27 - mul x27,x6,x4 - adcs x22,x22,x28 - mul x28,x7,x4 - adcs x23,x23,x0 - mul x0,x8,x4 - adcs x24,x24,x1 - mul x1,x9,x4 - adc x25,x25,x3 - mul x3,x10,x4 - adds x19,x19,x26 - umulh x26,x5,x4 - adcs x20,x20,x27 - umulh x27,x6,x4 - adcs x21,x21,x28 - umulh x28,x7,x4 - adcs x22,x22,x0 - umulh x0,x8,x4 - adcs x23,x23,x1 - umulh x1,x9,x4 - adcs x24,x24,x3 - umulh x3,x10,x4 - adc x25,x25,xzr - ldp x4,x2,[x29,#12*__SIZEOF_POINTER__] - - adds x11,x20,x26 - adcs x12,x21,x27 - adcs x13,x22,x28 - adcs x14,x23,x0 - adcs x15,x24,x1 - adcs x16,x25,x3 - + hint #29 ret ENDP @@ -2228,7 +1764,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mont_384|[FUNC] ALIGN 32 |sgn0_pty_mont_384| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2273,7 +1809,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP @@ -2282,7 +1818,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |sgn0_pty_mont_384x|[FUNC] ALIGN 32 |sgn0_pty_mont_384x| PROC - DCDU 3573752639 + hint #25 stp x29,x30,[sp,#-16*__SIZEOF_POINTER__]! add x29,sp,#0 stp x19,x20,[sp,#2*__SIZEOF_POINTER__] @@ -2371,7 +1907,7 @@ __SIZEOF_POINTER__ SETA 64/8 ldp x25,x26,[x29,#8*__SIZEOF_POINTER__] ldp x27,x28,[x29,#10*__SIZEOF_POINTER__] ldr x29,[sp],#16*__SIZEOF_POINTER__ - DCDU 3573752767 + hint #29 ret ENDP END diff --git a/blst_src/build/win64/mulq_mont_256-x86_64.asm b/blst_src/build/win64/mulq_mont_256-x86_64.asm index 70fa3c4b..9f30eb9b 100644 --- a/blst_src/build/win64/mulq_mont_256-x86_64.asm +++ b/blst_src/build/win64/mulq_mont_256-x86_64.asm @@ -1,8 +1,10 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ EXTERN mul_mont_sparse_256$1:NEAR EXTERN sqr_mont_sparse_256$1:NEAR EXTERN from_mont_256$1:NEAR EXTERN redc_mont_256$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS diff --git a/blst_src/build/win64/mulq_mont_384-x86_64.asm b/blst_src/build/win64/mulq_mont_384-x86_64.asm index 1c06947d..7e52c08a 100644 --- a/blst_src/build/win64/mulq_mont_384-x86_64.asm +++ b/blst_src/build/win64/mulq_mont_384-x86_64.asm @@ -1,4 +1,5 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ EXTERN mul_mont_384x$1:NEAR EXTERN sqr_mont_384x$1:NEAR EXTERN mul_382x$1:NEAR @@ -13,7 +14,7 @@ EXTERN mul_mont_384$1:NEAR EXTERN sqr_mont_384$1:NEAR EXTERN sqr_n_mul_mont_384$1:NEAR EXTERN sqr_n_mul_mont_383$1:NEAR -EXTERN sqr_mont_382x$1:NEAR +endif _DATA SEGMENT COMM __blst_platform_cap:DWORD:1 _DATA ENDS @@ -3190,890 +3191,124 @@ endif $L$SEH_end_sqr_n_mul_mont_383:: sqr_n_mul_mont_383 ENDP +.text$ ENDS +.pdata SEGMENT READONLY ALIGN(4) +ALIGN 4 + DD imagerel $L$SEH_begin_mul_mont_384x + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_prologue -ALIGN 32 -__mulq_mont_383_nonred PROC PRIVATE - DB 243,15,30,250 - - mov rbp,rax - mul r14 - mov r8,rax - mov rax,rbp - mov r9,rdx + DD imagerel $L$SEH_body_mul_mont_384x + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_body - mul r15 - add r9,rax - mov rax,rbp - adc rdx,0 - mov r10,rdx + DD imagerel $L$SEH_epilogue_mul_mont_384x + DD imagerel $L$SEH_end_mul_mont_384x + DD imagerel $L$SEH_info_mul_mont_384x_epilogue - mul r12 - add r10,rax - mov rax,rbp - adc rdx,0 - mov r11,rdx + DD imagerel $L$SEH_begin_sqr_mont_384x + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_prologue - mov r15,r8 - imul r8,QWORD PTR[8+rsp] + DD imagerel $L$SEH_body_sqr_mont_384x + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_body - mul r13 - add r11,rax - mov rax,rbp - adc rdx,0 - mov r12,rdx + DD imagerel $L$SEH_epilogue_sqr_mont_384x + DD imagerel $L$SEH_end_sqr_mont_384x + DD imagerel $L$SEH_info_sqr_mont_384x_epilogue - mul QWORD PTR[32+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - mov r13,rdx + DD imagerel $L$SEH_begin_mul_382x + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_info_mul_382x_prologue - mul QWORD PTR[40+rsi] - add r13,rax - mov rax,r8 - adc rdx,0 - mov r14,rdx + DD imagerel $L$SEH_body_mul_382x + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_info_mul_382x_body - mul QWORD PTR[rcx] - add r15,rax - mov rax,r8 - adc r15,rdx + DD imagerel $L$SEH_epilogue_mul_382x + DD imagerel $L$SEH_end_mul_382x + DD imagerel $L$SEH_info_mul_382x_epilogue - mul QWORD PTR[8+rcx] - add r9,rax - mov rax,r8 - adc rdx,0 - add r9,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_382x + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_prologue - mul QWORD PTR[16+rcx] - add r10,rax - mov rax,r8 - adc rdx,0 - add r10,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_382x + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_body - mul QWORD PTR[24+rcx] - add r11,r15 - adc rdx,0 - add r11,rax - mov rax,r8 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_sqr_382x + DD imagerel $L$SEH_end_sqr_382x + DD imagerel $L$SEH_info_sqr_382x_epilogue - mul QWORD PTR[32+rcx] - add r12,rax - mov rax,r8 - adc rdx,0 - add r12,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_mul_384 + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_info_mul_384_prologue - mul QWORD PTR[40+rcx] - add r13,rax - mov rax,QWORD PTR[8+rbx] - adc rdx,0 - add r13,r15 - adc r14,rdx + DD imagerel $L$SEH_body_mul_384 + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_info_mul_384_body - mov rbp,rax - mul QWORD PTR[rsi] - add r9,rax - mov rax,rbp - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_mul_384 + DD imagerel $L$SEH_end_mul_384 + DD imagerel $L$SEH_info_mul_384_epilogue - mul QWORD PTR[8+rsi] - add r10,rax - mov rax,rbp - adc rdx,0 - add r10,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_384 + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_info_sqr_384_prologue - mul QWORD PTR[16+rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - add r11,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_384 + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_info_sqr_384_body - mov r8,r9 - imul r9,QWORD PTR[8+rsp] + DD imagerel $L$SEH_epilogue_sqr_384 + DD imagerel $L$SEH_end_sqr_384 + DD imagerel $L$SEH_info_sqr_384_epilogue - mul QWORD PTR[24+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_begin_sqr_mont_384 + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_prologue - mul QWORD PTR[32+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r15 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_body_sqr_mont_384 + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_body - mul QWORD PTR[40+rsi] - add r14,r15 - adc rdx,0 - add r14,rax - mov rax,r9 - adc rdx,0 - mov r15,rdx + DD imagerel $L$SEH_epilogue_sqr_mont_384 + DD imagerel $L$SEH_end_sqr_mont_384 + DD imagerel $L$SEH_info_sqr_mont_384_epilogue - mul QWORD PTR[rcx] - add r8,rax - mov rax,r9 - adc r8,rdx + DD imagerel $L$SEH_begin_redc_mont_384 + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_prologue - mul QWORD PTR[8+rcx] - add r10,rax - mov rax,r9 - adc rdx,0 - add r10,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_body_redc_mont_384 + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_body - mul QWORD PTR[16+rcx] - add r11,rax - mov rax,r9 - adc rdx,0 - add r11,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_epilogue_redc_mont_384 + DD imagerel $L$SEH_end_redc_mont_384 + DD imagerel $L$SEH_info_redc_mont_384_epilogue - mul QWORD PTR[24+rcx] - add r12,r8 - adc rdx,0 - add r12,rax - mov rax,r9 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_begin_from_mont_384 + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_prologue - mul QWORD PTR[32+rcx] - add r13,rax - mov rax,r9 - adc rdx,0 - add r13,r8 - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_body_from_mont_384 + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_body - mul QWORD PTR[40+rcx] - add r14,rax - mov rax,QWORD PTR[16+rbx] - adc rdx,0 - add r14,r8 - adc r15,rdx + DD imagerel $L$SEH_epilogue_from_mont_384 + DD imagerel $L$SEH_end_from_mont_384 + DD imagerel $L$SEH_info_from_mont_384_epilogue - mov rbp,rax - mul QWORD PTR[rsi] - add r10,rax - mov rax,rbp - adc rdx,0 - mov r8,rdx + DD imagerel $L$SEH_begin_sgn0_pty_mont_384 + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_prologue - mul QWORD PTR[8+rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - add r11,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[16+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r8 - adc rdx,0 - mov r8,rdx - - mov r9,r10 - imul r10,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[32+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r8 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[40+rsi] - add r15,r8 - adc rdx,0 - add r15,rax - mov rax,r10 - adc rdx,0 - mov r8,rdx - - mul QWORD PTR[rcx] - add r9,rax - mov rax,r10 - adc r9,rdx - - mul QWORD PTR[8+rcx] - add r11,rax - mov rax,r10 - adc rdx,0 - add r11,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[16+rcx] - add r12,rax - mov rax,r10 - adc rdx,0 - add r12,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[24+rcx] - add r13,r9 - adc rdx,0 - add r13,rax - mov rax,r10 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[32+rcx] - add r14,rax - mov rax,r10 - adc rdx,0 - add r14,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[40+rcx] - add r15,rax - mov rax,QWORD PTR[24+rbx] - adc rdx,0 - add r15,r9 - adc r8,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[8+rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - add r12,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[16+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r9 - adc rdx,0 - mov r9,rdx - - mov r10,r11 - imul r11,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[32+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r9 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[40+rsi] - add r8,r9 - adc rdx,0 - add r8,rax - mov rax,r11 - adc rdx,0 - mov r9,rdx - - mul QWORD PTR[rcx] - add r10,rax - mov rax,r11 - adc r10,rdx - - mul QWORD PTR[8+rcx] - add r12,rax - mov rax,r11 - adc rdx,0 - add r12,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[16+rcx] - add r13,rax - mov rax,r11 - adc rdx,0 - add r13,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[24+rcx] - add r14,r10 - adc rdx,0 - add r14,rax - mov rax,r11 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[32+rcx] - add r15,rax - mov rax,r11 - adc rdx,0 - add r15,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[40+rcx] - add r8,rax - mov rax,QWORD PTR[32+rbx] - adc rdx,0 - add r8,r10 - adc r9,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r12,rax - mov rax,rbp - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[8+rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - add r13,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[16+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r10 - adc rdx,0 - mov r10,rdx - - mov r11,r12 - imul r12,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[32+rsi] - add r8,rax - mov rax,rbp - adc rdx,0 - add r8,r10 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[40+rsi] - add r9,r10 - adc rdx,0 - add r9,rax - mov rax,r12 - adc rdx,0 - mov r10,rdx - - mul QWORD PTR[rcx] - add r11,rax - mov rax,r12 - adc r11,rdx - - mul QWORD PTR[8+rcx] - add r13,rax - mov rax,r12 - adc rdx,0 - add r13,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[16+rcx] - add r14,rax - mov rax,r12 - adc rdx,0 - add r14,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[24+rcx] - add r15,r11 - adc rdx,0 - add r15,rax - mov rax,r12 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[32+rcx] - add r8,rax - mov rax,r12 - adc rdx,0 - add r8,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[40+rcx] - add r9,rax - mov rax,QWORD PTR[40+rbx] - adc rdx,0 - add r9,r11 - adc r10,rdx - - mov rbp,rax - mul QWORD PTR[rsi] - add r13,rax - mov rax,rbp - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[8+rsi] - add r14,rax - mov rax,rbp - adc rdx,0 - add r14,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[16+rsi] - add r15,rax - mov rax,rbp - adc rdx,0 - add r15,r11 - adc rdx,0 - mov r11,rdx - - mov r12,r13 - imul r13,QWORD PTR[8+rsp] - - mul QWORD PTR[24+rsi] - add r8,rax - mov rax,rbp - adc rdx,0 - add r8,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[32+rsi] - add r9,rax - mov rax,rbp - adc rdx,0 - add r9,r11 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[40+rsi] - add r10,r11 - adc rdx,0 - add r10,rax - mov rax,r13 - adc rdx,0 - mov r11,rdx - - mul QWORD PTR[rcx] - add r12,rax - mov rax,r13 - adc r12,rdx - - mul QWORD PTR[8+rcx] - add r14,rax - mov rax,r13 - adc rdx,0 - add r14,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[16+rcx] - add r15,rax - mov rax,r13 - adc rdx,0 - add r15,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[24+rcx] - add r8,r12 - adc rdx,0 - add r8,rax - mov rax,r13 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[32+rcx] - add r9,rax - mov rax,r13 - adc rdx,0 - add r9,r12 - adc rdx,0 - mov r12,rdx - - mul QWORD PTR[40+rcx] - add r10,rax - mov rax,r14 - adc rdx,0 - add r10,r12 - adc r11,rdx - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif -__mulq_mont_383_nonred ENDP -PUBLIC sqr_mont_382x - - -ALIGN 32 -sqr_mont_382x PROC PUBLIC - DB 243,15,30,250 - mov QWORD PTR[8+rsp],rdi ;WIN64 prologue - mov QWORD PTR[16+rsp],rsi - mov r11,rsp -$L$SEH_begin_sqr_mont_382x:: - - - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 -ifdef __BLST_PORTABLE__ - test DWORD PTR[__blst_platform_cap],1 - jnz sqr_mont_382x$1 -endif - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,136 - -$L$SEH_body_sqr_mont_382x:: - - - mov QWORD PTR[rsp],rcx - mov rcx,rdx - mov QWORD PTR[16+rsp],rsi - mov QWORD PTR[24+rsp],rdi - - - mov r8,QWORD PTR[rsi] - mov r9,QWORD PTR[8+rsi] - mov r10,QWORD PTR[16+rsi] - mov r11,QWORD PTR[24+rsi] - mov r12,QWORD PTR[32+rsi] - mov r13,QWORD PTR[40+rsi] - - mov r14,r8 - add r8,QWORD PTR[48+rsi] - mov r15,r9 - adc r9,QWORD PTR[56+rsi] - mov rax,r10 - adc r10,QWORD PTR[64+rsi] - mov rdx,r11 - adc r11,QWORD PTR[72+rsi] - mov rbx,r12 - adc r12,QWORD PTR[80+rsi] - mov rbp,r13 - adc r13,QWORD PTR[88+rsi] - - sub r14,QWORD PTR[48+rsi] - sbb r15,QWORD PTR[56+rsi] - sbb rax,QWORD PTR[64+rsi] - sbb rdx,QWORD PTR[72+rsi] - sbb rbx,QWORD PTR[80+rsi] - sbb rbp,QWORD PTR[88+rsi] - sbb rdi,rdi - - mov QWORD PTR[((32+0))+rsp],r8 - mov QWORD PTR[((32+8))+rsp],r9 - mov QWORD PTR[((32+16))+rsp],r10 - mov QWORD PTR[((32+24))+rsp],r11 - mov QWORD PTR[((32+32))+rsp],r12 - mov QWORD PTR[((32+40))+rsp],r13 - - mov QWORD PTR[((32+48))+rsp],r14 - mov QWORD PTR[((32+56))+rsp],r15 - mov QWORD PTR[((32+64))+rsp],rax - mov QWORD PTR[((32+72))+rsp],rdx - mov QWORD PTR[((32+80))+rsp],rbx - mov QWORD PTR[((32+88))+rsp],rbp - mov QWORD PTR[((32+96))+rsp],rdi - - - - lea rbx,QWORD PTR[48+rsi] - - mov rax,QWORD PTR[48+rsi] - mov r14,QWORD PTR[rsi] - mov r15,QWORD PTR[8+rsi] - mov r12,QWORD PTR[16+rsi] - mov r13,QWORD PTR[24+rsi] - - mov rdi,QWORD PTR[24+rsp] - call __mulq_mont_383_nonred - add r14,r14 - adc r15,r15 - adc r8,r8 - adc r9,r9 - adc r10,r10 - adc r11,r11 - - mov QWORD PTR[48+rdi],r14 - mov QWORD PTR[56+rdi],r15 - mov QWORD PTR[64+rdi],r8 - mov QWORD PTR[72+rdi],r9 - mov QWORD PTR[80+rdi],r10 - mov QWORD PTR[88+rdi],r11 - - lea rsi,QWORD PTR[32+rsp] - lea rbx,QWORD PTR[((32+48))+rsp] - - mov rax,QWORD PTR[((32+48))+rsp] - mov r14,QWORD PTR[((32+0))+rsp] - mov r15,QWORD PTR[((32+8))+rsp] - mov r12,QWORD PTR[((32+16))+rsp] - mov r13,QWORD PTR[((32+24))+rsp] - - call __mulq_mont_383_nonred - mov rsi,QWORD PTR[((32+96))+rsp] - mov r12,QWORD PTR[((32+0))+rsp] - mov r13,QWORD PTR[((32+8))+rsp] - and r12,rsi - mov rax,QWORD PTR[((32+16))+rsp] - and r13,rsi - mov rbx,QWORD PTR[((32+24))+rsp] - and rax,rsi - mov rbp,QWORD PTR[((32+32))+rsp] - and rbx,rsi - and rbp,rsi - and rsi,QWORD PTR[((32+40))+rsp] - - sub r14,r12 - mov r12,QWORD PTR[rcx] - sbb r15,r13 - mov r13,QWORD PTR[8+rcx] - sbb r8,rax - mov rax,QWORD PTR[16+rcx] - sbb r9,rbx - mov rbx,QWORD PTR[24+rcx] - sbb r10,rbp - mov rbp,QWORD PTR[32+rcx] - sbb r11,rsi - sbb rsi,rsi - - and r12,rsi - and r13,rsi - and rax,rsi - and rbx,rsi - and rbp,rsi - and rsi,QWORD PTR[40+rcx] - - add r14,r12 - adc r15,r13 - adc r8,rax - adc r9,rbx - adc r10,rbp - adc r11,rsi - - mov QWORD PTR[rdi],r14 - mov QWORD PTR[8+rdi],r15 - mov QWORD PTR[16+rdi],r8 - mov QWORD PTR[24+rdi],r9 - mov QWORD PTR[32+rdi],r10 - mov QWORD PTR[40+rdi],r11 - lea r8,QWORD PTR[136+rsp] - mov r15,QWORD PTR[r8] - - mov r14,QWORD PTR[8+r8] - - mov r13,QWORD PTR[16+r8] - - mov r12,QWORD PTR[24+r8] - - mov rbx,QWORD PTR[32+r8] - - mov rbp,QWORD PTR[40+r8] - - lea rsp,QWORD PTR[48+r8] - -$L$SEH_epilogue_sqr_mont_382x:: - mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue - mov rsi,QWORD PTR[16+rsp] - - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif - -$L$SEH_end_sqr_mont_382x:: -sqr_mont_382x ENDP -.text$ ENDS -.pdata SEGMENT READONLY ALIGN(4) -ALIGN 4 - DD imagerel $L$SEH_begin_mul_mont_384x - DD imagerel $L$SEH_body_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_prologue - - DD imagerel $L$SEH_body_mul_mont_384x - DD imagerel $L$SEH_epilogue_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_body - - DD imagerel $L$SEH_epilogue_mul_mont_384x - DD imagerel $L$SEH_end_mul_mont_384x - DD imagerel $L$SEH_info_mul_mont_384x_epilogue - - DD imagerel $L$SEH_begin_sqr_mont_384x - DD imagerel $L$SEH_body_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_prologue - - DD imagerel $L$SEH_body_sqr_mont_384x - DD imagerel $L$SEH_epilogue_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_body - - DD imagerel $L$SEH_epilogue_sqr_mont_384x - DD imagerel $L$SEH_end_sqr_mont_384x - DD imagerel $L$SEH_info_sqr_mont_384x_epilogue - - DD imagerel $L$SEH_begin_mul_382x - DD imagerel $L$SEH_body_mul_382x - DD imagerel $L$SEH_info_mul_382x_prologue - - DD imagerel $L$SEH_body_mul_382x - DD imagerel $L$SEH_epilogue_mul_382x - DD imagerel $L$SEH_info_mul_382x_body - - DD imagerel $L$SEH_epilogue_mul_382x - DD imagerel $L$SEH_end_mul_382x - DD imagerel $L$SEH_info_mul_382x_epilogue - - DD imagerel $L$SEH_begin_sqr_382x - DD imagerel $L$SEH_body_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_prologue - - DD imagerel $L$SEH_body_sqr_382x - DD imagerel $L$SEH_epilogue_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_body - - DD imagerel $L$SEH_epilogue_sqr_382x - DD imagerel $L$SEH_end_sqr_382x - DD imagerel $L$SEH_info_sqr_382x_epilogue - - DD imagerel $L$SEH_begin_mul_384 - DD imagerel $L$SEH_body_mul_384 - DD imagerel $L$SEH_info_mul_384_prologue - - DD imagerel $L$SEH_body_mul_384 - DD imagerel $L$SEH_epilogue_mul_384 - DD imagerel $L$SEH_info_mul_384_body - - DD imagerel $L$SEH_epilogue_mul_384 - DD imagerel $L$SEH_end_mul_384 - DD imagerel $L$SEH_info_mul_384_epilogue - - DD imagerel $L$SEH_begin_sqr_384 - DD imagerel $L$SEH_body_sqr_384 - DD imagerel $L$SEH_info_sqr_384_prologue - - DD imagerel $L$SEH_body_sqr_384 - DD imagerel $L$SEH_epilogue_sqr_384 - DD imagerel $L$SEH_info_sqr_384_body - - DD imagerel $L$SEH_epilogue_sqr_384 - DD imagerel $L$SEH_end_sqr_384 - DD imagerel $L$SEH_info_sqr_384_epilogue - - DD imagerel $L$SEH_begin_sqr_mont_384 - DD imagerel $L$SEH_body_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_prologue - - DD imagerel $L$SEH_body_sqr_mont_384 - DD imagerel $L$SEH_epilogue_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_body - - DD imagerel $L$SEH_epilogue_sqr_mont_384 - DD imagerel $L$SEH_end_sqr_mont_384 - DD imagerel $L$SEH_info_sqr_mont_384_epilogue - - DD imagerel $L$SEH_begin_redc_mont_384 - DD imagerel $L$SEH_body_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_prologue - - DD imagerel $L$SEH_body_redc_mont_384 - DD imagerel $L$SEH_epilogue_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_body - - DD imagerel $L$SEH_epilogue_redc_mont_384 - DD imagerel $L$SEH_end_redc_mont_384 - DD imagerel $L$SEH_info_redc_mont_384_epilogue - - DD imagerel $L$SEH_begin_from_mont_384 - DD imagerel $L$SEH_body_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_prologue - - DD imagerel $L$SEH_body_from_mont_384 - DD imagerel $L$SEH_epilogue_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_body - - DD imagerel $L$SEH_epilogue_from_mont_384 - DD imagerel $L$SEH_end_from_mont_384 - DD imagerel $L$SEH_info_from_mont_384_epilogue - - DD imagerel $L$SEH_begin_sgn0_pty_mont_384 - DD imagerel $L$SEH_body_sgn0_pty_mont_384 - DD imagerel $L$SEH_info_sgn0_pty_mont_384_prologue - - DD imagerel $L$SEH_body_sgn0_pty_mont_384 - DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 - DD imagerel $L$SEH_info_sgn0_pty_mont_384_body + DD imagerel $L$SEH_body_sgn0_pty_mont_384 + DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 + DD imagerel $L$SEH_info_sgn0_pty_mont_384_body DD imagerel $L$SEH_epilogue_sgn0_pty_mont_384 DD imagerel $L$SEH_end_sgn0_pty_mont_384 @@ -4127,18 +3362,6 @@ ALIGN 4 DD imagerel $L$SEH_end_sqr_n_mul_mont_383 DD imagerel $L$SEH_info_sqr_n_mul_mont_383_epilogue - DD imagerel $L$SEH_begin_sqr_mont_382x - DD imagerel $L$SEH_body_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_prologue - - DD imagerel $L$SEH_body_sqr_mont_382x - DD imagerel $L$SEH_epilogue_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_body - - DD imagerel $L$SEH_epilogue_sqr_mont_382x - DD imagerel $L$SEH_end_sqr_mont_382x - DD imagerel $L$SEH_info_sqr_mont_382x_epilogue - .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 @@ -4502,32 +3725,6 @@ DB 000h,074h,001h,000h DB 000h,064h,002h,000h DB 000h,000h,000h,000h -$L$SEH_info_sqr_mont_382x_prologue:: -DB 1,0,5,00bh -DB 0,074h,1,0 -DB 0,064h,2,0 -DB 0,0b3h -DB 0,0 - DD 0,0 -$L$SEH_info_sqr_mont_382x_body:: -DB 1,0,18,0 -DB 000h,0f4h,011h,000h -DB 000h,0e4h,012h,000h -DB 000h,0d4h,013h,000h -DB 000h,0c4h,014h,000h -DB 000h,034h,015h,000h -DB 000h,054h,016h,000h -DB 000h,074h,018h,000h -DB 000h,064h,019h,000h -DB 000h,001h,017h,000h -DB 000h,000h,000h,000h -DB 000h,000h,000h,000h -$L$SEH_info_sqr_mont_382x_epilogue:: -DB 1,0,4,0 -DB 000h,074h,001h,000h -DB 000h,064h,002h,000h -DB 000h,000h,000h,000h - .xdata ENDS END diff --git a/blst_src/build/win64/mulx_mont_256-x86_64.asm b/blst_src/build/win64/mulx_mont_256-x86_64.asm index 19946a28..f1dd4e46 100644 --- a/blst_src/build/win64/mulx_mont_256-x86_64.asm +++ b/blst_src/build/win64/mulx_mont_256-x86_64.asm @@ -1,8 +1,10 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ PUBLIC mul_mont_sparse_256$1 PUBLIC sqr_mont_sparse_256$1 PUBLIC from_mont_256$1 PUBLIC redc_mont_256$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' PUBLIC mulx_mont_sparse_256 @@ -219,17 +221,14 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rbp - adox r14,r9 - adcx r14,r10 - adox r15,r10 + adox r9,r10 + adcx r14,r9 adcx r15,r10 - adox r10,r10 - adc r10,0 mov r11,rax imul rax,r8 - xor rbp,rbp + xor r10,r10 mulx r9,rbp,QWORD PTR[((0+128))+rsi] adox r12,rbp adcx r13,r9 @@ -264,17 +263,14 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rbp - adox r15,r9 - adcx r15,r11 - adox r10,r11 + adox r9,r11 + adcx r15,r9 adcx r10,r11 - adox r11,r11 - adc r11,0 mov r12,rax imul rax,r8 - xor rbp,rbp + xor r11,r11 mulx r9,rbp,QWORD PTR[((0+128))+rsi] adox r13,rbp adcx r14,r9 @@ -309,16 +305,13 @@ __mulx_mont_sparse_256 PROC PRIVATE mulx r9,rbp,QWORD PTR[((24+128))+rcx] mov rdx,rax adcx r15,rbp - adox r10,r9 - adcx r10,r12 - adox r11,r12 + adox r9,r12 + adcx r10,r9 adcx r11,r12 - adox r12,r12 - adc r12,0 imul rdx,r8 - xor rbp,rbp + xor r12,r12 mulx r9,r13,QWORD PTR[((0+128))+rcx] adcx r13,rax adox r14,r9 @@ -335,11 +328,10 @@ __mulx_mont_sparse_256 PROC PRIVATE mov rdx,r14 lea rcx,QWORD PTR[128+rcx] adcx r10,rbp - adox r11,r9 + adox r9,r13 mov rax,r15 - adcx r11,r13 - adox r12,r13 - adc r12,0 + adcx r11,r9 + adcx r12,r13 diff --git a/blst_src/build/win64/mulx_mont_384-x86_64.asm b/blst_src/build/win64/mulx_mont_384-x86_64.asm index 8a6752f2..02b820ad 100644 --- a/blst_src/build/win64/mulx_mont_384-x86_64.asm +++ b/blst_src/build/win64/mulx_mont_384-x86_64.asm @@ -1,4 +1,5 @@ OPTION DOTNAME +ifdef __BLST_PORTABLE__ PUBLIC mul_mont_384x$1 PUBLIC sqr_mont_384x$1 PUBLIC mul_382x$1 @@ -13,7 +14,7 @@ PUBLIC mul_mont_384$1 PUBLIC sqr_mont_384$1 PUBLIC sqr_n_mul_mont_384$1 PUBLIC sqr_n_mul_mont_383$1 -PUBLIC sqr_mont_382x$1 +endif .text$ SEGMENT ALIGN(256) 'CODE' @@ -2091,8 +2092,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r8 adox r14,rdi - adcx r15,rbp - adox r15,rax + adcx rbp,rax + adox r15,rbp adox rax,rax @@ -2120,11 +2121,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rdi - adox r14,rbp - adcx r14,r8 - adox r15,r8 + adox rbp,r8 + adcx r14,rbp adcx r15,r8 - adox rax,r8 adcx rax,r8 mov QWORD PTR[16+rsp],r9 imul r9,QWORD PTR[8+rsp] @@ -2154,8 +2153,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r9 adox r15,rdi - adcx rax,rbp - adox rax,r8 + adcx rbp,r8 + adox rax,rbp adox r8,r8 @@ -2183,11 +2182,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rdi - adox r15,rbp - adcx r15,r9 - adox rax,r9 + adox rbp,r9 + adcx r15,rbp adcx rax,r9 - adox r8,r9 adcx r8,r9 mov QWORD PTR[16+rsp],r10 imul r10,QWORD PTR[8+rsp] @@ -2217,8 +2214,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r10 adox rax,rdi - adcx r8,rbp - adox r8,r9 + adcx rbp,r9 + adox r8,rbp adox r9,r9 @@ -2246,11 +2243,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[32+rbx] adcx r15,rdi - adox rax,rbp - adcx rax,r10 - adox r8,r10 + adox rbp,r10 + adcx rax,rbp adcx r8,r10 - adox r9,r10 adcx r9,r10 mov QWORD PTR[16+rsp],r11 imul r11,QWORD PTR[8+rsp] @@ -2280,8 +2275,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r11 adox r8,rdi - adcx r9,rbp - adox r9,r10 + adcx rbp,r10 + adox r9,rbp adox r10,r10 @@ -2309,11 +2304,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[40+rbx] adcx rax,rdi - adox r8,rbp - adcx r8,r11 - adox r9,r11 + adox rbp,r11 + adcx r8,rbp adcx r9,r11 - adox r10,r11 adcx r10,r11 mov QWORD PTR[16+rsp],r12 imul r12,QWORD PTR[8+rsp] @@ -2343,8 +2336,8 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rsi] mov rdx,r12 adox r9,rdi - adcx r10,rbp - adox r10,r11 + adcx rbp,r11 + adox r10,rbp adox r11,r11 @@ -2372,11 +2365,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,r13 adcx r8,rdi - adox r9,rbp - adcx r9,r12 - adox r10,r12 + adox rbp,r12 + adcx r9,rbp adcx r10,r12 - adox r11,r12 adcx r11,r12 imul rdx,QWORD PTR[8+rsp] mov rbx,QWORD PTR[24+rsp] @@ -2407,10 +2398,9 @@ __mulx_mont_384 PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] adcx r9,rdi - adox r10,rbp + adox rbp,r12 mov rdx,r14 - adcx r10,r12 - adox r11,r12 + adcx r10,rbp lea rcx,QWORD PTR[128+rcx] mov r12,r8 adc r11,0 @@ -2819,9 +2809,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[16+rbx] adcx r13,rdi - adox r14,rbp - adcx r14,rax - adox r15,rax + adox rbp,rax + adcx r14,rbp adcx r15,rax mov r8,r9 imul r9,QWORD PTR[8+rsp] @@ -2879,9 +2868,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[24+rbx] adcx r14,rdi - adox r15,rbp - adcx r15,r8 - adox rax,r8 + adox rbp,r8 + adcx r15,rbp adcx rax,r8 mov r9,r10 imul r10,QWORD PTR[8+rsp] @@ -2939,9 +2927,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[32+rbx] adcx r15,rdi - adox rax,rbp - adcx rax,r9 - adox r8,r9 + adox rbp,r9 + adcx rax,rbp adcx r8,r9 mov r10,r11 imul r11,QWORD PTR[8+rsp] @@ -2999,9 +2986,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,QWORD PTR[40+rbx] adcx rax,rdi - adox r8,rbp - adcx r8,r10 - adox r9,r10 + adox rbp,r10 + adcx r8,rbp adcx r9,r10 mov r11,r12 imul r12,QWORD PTR[8+rsp] @@ -3059,9 +3045,8 @@ __mulx_mont_383_nonred PROC PRIVATE mulx rbp,rdi,QWORD PTR[((40+128))+rcx] mov rdx,r13 adcx r8,rdi - adox r9,rbp - adcx r9,r11 - adox r10,r11 + adox rbp,r11 + adcx r9,rbp adcx r10,r11 imul rdx,QWORD PTR[8+rsp] mov rbx,QWORD PTR[24+rsp] @@ -3115,215 +3100,6 @@ else endif __mulx_mont_383_nonred ENDP -PUBLIC sqrx_mont_382x - - -ALIGN 32 -sqrx_mont_382x PROC PUBLIC - DB 243,15,30,250 - mov QWORD PTR[8+rsp],rdi ;WIN64 prologue - mov QWORD PTR[16+rsp],rsi - mov r11,rsp -$L$SEH_begin_sqrx_mont_382x:: - - - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - mov rcx,r9 -sqr_mont_382x$1:: - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,136 - -$L$SEH_body_sqrx_mont_382x:: - - - mov QWORD PTR[rsp],rcx - mov rcx,rdx - mov QWORD PTR[16+rsp],rdi - mov QWORD PTR[24+rsp],rsi - - -ifdef __SGX_LVI_HARDENING__ - lfence -endif - mov r8,QWORD PTR[rsi] - mov r9,QWORD PTR[8+rsi] - mov r10,QWORD PTR[16+rsi] - mov r11,QWORD PTR[24+rsi] - mov r12,QWORD PTR[32+rsi] - mov r13,QWORD PTR[40+rsi] - - mov r14,r8 - add r8,QWORD PTR[48+rsi] - mov r15,r9 - adc r9,QWORD PTR[56+rsi] - mov rax,r10 - adc r10,QWORD PTR[64+rsi] - mov rdx,r11 - adc r11,QWORD PTR[72+rsi] - mov rbx,r12 - adc r12,QWORD PTR[80+rsi] - mov rbp,r13 - adc r13,QWORD PTR[88+rsi] - - sub r14,QWORD PTR[48+rsi] - sbb r15,QWORD PTR[56+rsi] - sbb rax,QWORD PTR[64+rsi] - sbb rdx,QWORD PTR[72+rsi] - sbb rbx,QWORD PTR[80+rsi] - sbb rbp,QWORD PTR[88+rsi] - sbb rdi,rdi - - mov QWORD PTR[((32+0))+rsp],r8 - mov QWORD PTR[((32+8))+rsp],r9 - mov QWORD PTR[((32+16))+rsp],r10 - mov QWORD PTR[((32+24))+rsp],r11 - mov QWORD PTR[((32+32))+rsp],r12 - mov QWORD PTR[((32+40))+rsp],r13 - - mov QWORD PTR[((32+48))+rsp],r14 - mov QWORD PTR[((32+56))+rsp],r15 - mov QWORD PTR[((32+64))+rsp],rax - mov QWORD PTR[((32+72))+rsp],rdx - mov QWORD PTR[((32+80))+rsp],rbx - mov QWORD PTR[((32+88))+rsp],rbp - mov QWORD PTR[((32+96))+rsp],rdi - - - - lea rbx,QWORD PTR[48+rsi] - - mov rdx,QWORD PTR[48+rsi] - mov r14,QWORD PTR[rsi] - mov r15,QWORD PTR[8+rsi] - mov rax,QWORD PTR[16+rsi] - mov r12,QWORD PTR[24+rsi] - mov rdi,QWORD PTR[32+rsi] - mov rbp,QWORD PTR[40+rsi] - lea rsi,QWORD PTR[((-128))+rsi] - lea rcx,QWORD PTR[((-128))+rcx] - - mulx r9,r8,r14 - call __mulx_mont_383_nonred - add rdx,rdx - adc r15,r15 - adc rax,rax - adc r12,r12 - adc rdi,rdi - adc rbp,rbp - - mov QWORD PTR[48+rbx],rdx - mov QWORD PTR[56+rbx],r15 - mov QWORD PTR[64+rbx],rax - mov QWORD PTR[72+rbx],r12 - mov QWORD PTR[80+rbx],rdi - mov QWORD PTR[88+rbx],rbp - - lea rsi,QWORD PTR[((32-128))+rsp] - lea rbx,QWORD PTR[((32+48))+rsp] - - mov rdx,QWORD PTR[((32+48))+rsp] - mov r14,QWORD PTR[((32+0))+rsp] - mov r15,QWORD PTR[((32+8))+rsp] - mov rax,QWORD PTR[((32+16))+rsp] - mov r12,QWORD PTR[((32+24))+rsp] - mov rdi,QWORD PTR[((32+32))+rsp] - mov rbp,QWORD PTR[((32+40))+rsp] - - - - mulx r9,r8,r14 - call __mulx_mont_383_nonred - mov r14,QWORD PTR[((32+96))+rsp] - lea rcx,QWORD PTR[128+rcx] - mov r8,QWORD PTR[((32+0))+rsp] - and r8,r14 - mov r9,QWORD PTR[((32+8))+rsp] - and r9,r14 - mov r10,QWORD PTR[((32+16))+rsp] - and r10,r14 - mov r11,QWORD PTR[((32+24))+rsp] - and r11,r14 - mov r13,QWORD PTR[((32+32))+rsp] - and r13,r14 - and r14,QWORD PTR[((32+40))+rsp] - - sub rdx,r8 - mov r8,QWORD PTR[rcx] - sbb r15,r9 - mov r9,QWORD PTR[8+rcx] - sbb rax,r10 - mov r10,QWORD PTR[16+rcx] - sbb r12,r11 - mov r11,QWORD PTR[24+rcx] - sbb rdi,r13 - mov r13,QWORD PTR[32+rcx] - sbb rbp,r14 - sbb r14,r14 - - and r8,r14 - and r9,r14 - and r10,r14 - and r11,r14 - and r13,r14 - and r14,QWORD PTR[40+rcx] - - add rdx,r8 - adc r15,r9 - adc rax,r10 - adc r12,r11 - adc rdi,r13 - adc rbp,r14 - - mov QWORD PTR[rbx],rdx - mov QWORD PTR[8+rbx],r15 - mov QWORD PTR[16+rbx],rax - mov QWORD PTR[24+rbx],r12 - mov QWORD PTR[32+rbx],rdi - mov QWORD PTR[40+rbx],rbp - lea r8,QWORD PTR[136+rsp] - mov r15,QWORD PTR[r8] - - mov r14,QWORD PTR[8+r8] - - mov r13,QWORD PTR[16+r8] - - mov r12,QWORD PTR[24+r8] - - mov rbx,QWORD PTR[32+r8] - - mov rbp,QWORD PTR[40+r8] - - lea rsp,QWORD PTR[48+r8] - -$L$SEH_epilogue_sqrx_mont_382x:: - mov rdi,QWORD PTR[8+rsp] ;WIN64 epilogue - mov rsi,QWORD PTR[16+rsp] - - -ifdef __SGX_LVI_HARDENING__ - pop rdx - lfence - jmp rdx - ud2 -else - DB 0F3h,0C3h -endif - -$L$SEH_end_sqrx_mont_382x:: -sqrx_mont_382x ENDP .text$ ENDS .pdata SEGMENT READONLY ALIGN(4) ALIGN 4 @@ -3495,18 +3271,6 @@ ALIGN 4 DD imagerel $L$SEH_end_sqrx_n_mul_mont_383 DD imagerel $L$SEH_info_sqrx_n_mul_mont_383_epilogue - DD imagerel $L$SEH_begin_sqrx_mont_382x - DD imagerel $L$SEH_body_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_prologue - - DD imagerel $L$SEH_body_sqrx_mont_382x - DD imagerel $L$SEH_epilogue_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_body - - DD imagerel $L$SEH_epilogue_sqrx_mont_382x - DD imagerel $L$SEH_end_sqrx_mont_382x - DD imagerel $L$SEH_info_sqrx_mont_382x_epilogue - .pdata ENDS .xdata SEGMENT READONLY ALIGN(8) ALIGN 8 @@ -3874,32 +3638,6 @@ DB 000h,074h,001h,000h DB 000h,064h,002h,000h DB 000h,000h,000h,000h -$L$SEH_info_sqrx_mont_382x_prologue:: -DB 1,0,5,00bh -DB 0,074h,1,0 -DB 0,064h,2,0 -DB 0,0b3h -DB 0,0 - DD 0,0 -$L$SEH_info_sqrx_mont_382x_body:: -DB 1,0,18,0 -DB 000h,0f4h,011h,000h -DB 000h,0e4h,012h,000h -DB 000h,0d4h,013h,000h -DB 000h,0c4h,014h,000h -DB 000h,034h,015h,000h -DB 000h,054h,016h,000h -DB 000h,074h,018h,000h -DB 000h,064h,019h,000h -DB 000h,001h,017h,000h -DB 000h,000h,000h,000h -DB 000h,000h,000h,000h -$L$SEH_info_sqrx_mont_382x_epilogue:: -DB 1,0,4,0 -DB 000h,074h,001h,000h -DB 000h,064h,002h,000h -DB 000h,000h,000h,000h - .xdata ENDS END diff --git a/blst_src/build/win64/sha256-armv8.asm b/blst_src/build/win64/sha256-armv8.asm index 46e821e4..83e4e99c 100644 --- a/blst_src/build/win64/sha256-armv8.asm +++ b/blst_src/build/win64/sha256-armv8.asm @@ -49,6 +49,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_block_armv8|[FUNC] ALIGN 64 |blst_sha256_block_armv8| PROC + hint #34 |$Lv8_entry| stp x29,x30,[sp,#-2*__SIZEOF_POINTER__]! add x29,sp,#0 @@ -189,6 +190,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_block_data_order|[FUNC] ALIGN 16 |blst_sha256_block_data_order| PROC + hint #34 adrp x16,__blst_platform_cap ldr w16,[x16,__blst_platform_cap] tst w16,#1 @@ -1038,6 +1040,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_emit|[FUNC] ALIGN 16 |blst_sha256_emit| PROC + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] if :lnot::def: __AARCH64EB__ @@ -1066,6 +1069,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_bcopy|[FUNC] ALIGN 16 |blst_sha256_bcopy| PROC + hint #34 |$Loop_bcopy| ldrb w3,[x1],#1 sub x2,x2,#1 @@ -1079,6 +1083,7 @@ __SIZEOF_POINTER__ SETA 64/8 EXPORT |blst_sha256_hcopy|[FUNC] ALIGN 16 |blst_sha256_hcopy| PROC + hint #34 ldp x4,x5,[x1] ldp x6,x7,[x1,#16] stp x4,x5,[x0] diff --git a/blst_src/cpuid.c b/blst_src/cpuid.c index 82317043..a1a01d4a 100644 --- a/blst_src/cpuid.c +++ b/blst_src/cpuid.c @@ -47,14 +47,14 @@ static int __blst_cpuid(void) return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # elif defined(__SUNPRO_C) # pragma init(__blst_cpuid) # endif -#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__aarch64) || defined(_M_ARM64) || defined(_M_ARM64EC) # if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) extern unsigned long getauxval(unsigned long type) __attribute__ ((weak)); @@ -105,10 +105,56 @@ static int __blst_cpuid(void) return 0; } -# if defined(_MSC_VER) && !defined(__clang__) +# if defined(_MSC_VER) && !defined(__clang__) && !defined(__BLST_DLL_MAIN__) # pragma section(".CRT$XCU",read) __declspec(allocate(".CRT$XCU")) static int (*p)(void) = __blst_cpuid; # endif # endif #endif + +#if defined(_WIN64) && defined(__BLST_DLL_MAIN__) +# define IsProcessorFeaturePresent mask_IsProcessorFeaturePresent +# define WIN32_LEAN_AND_MEAN +# include + +BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD dwReason, LPVOID lpvReserved) +{ + if (dwReason == DLL_PROCESS_ATTACH) { + DisableThreadLibraryCalls(hinstDLL); + __blst_cpuid(); + } + + return TRUE; + + (void)lpvReserved; +} + +# if defined(_MSC_VER) +/* + * Even though we don't have memcpy/memset anywhere, MSVC compiler + * generates calls to them as it recognizes corresponding patterns. + */ +#pragma function(memcpy) +void *memcpy(unsigned char *dst, const unsigned char *src, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = *src++; + + return ret; +} + +#pragma function(memset) +void *memset(unsigned char *dst, int c, size_t n) +{ + void *ret = dst; + + while(n--) + *dst++ = (unsigned char)c; + + return ret; +} +# endif +#endif diff --git a/blst_src/ec_mult.h b/blst_src/ec_mult.h index 24c151ba..9182c7bf 100644 --- a/blst_src/ec_mult.h +++ b/blst_src/ec_mult.h @@ -210,8 +210,7 @@ static void ptype##_mult_w##SZ(ptype *ret, const ptype *point, \ wval &= wmask; \ wval = booth_encode(wval, SZ); \ row_is_inf = ptype##_gather_booth_w##SZ(row, table, wval); \ - if (bits > 0) ptype##_add(sum, ret, row); \ - else ptype##_dadd(sum, ret, row, NULL); \ + ptype##_dadd(sum, ret, row, NULL); \ ptype##_ccopy(ret, sum, (ret_is_inf | row_is_inf) ^ 1); \ sum_is_inf = vec_is_zero(ret->Z, sizeof(ret->Z)); \ ret_is_inf |= sum_is_inf; \ diff --git a/blst_src/exports.c b/blst_src/exports.c index 1ca4d475..359ed3d1 100644 --- a/blst_src/exports.c +++ b/blst_src/exports.c @@ -64,7 +64,7 @@ void blst_fr_cneg(vec256 ret, const vec256 a, int flag) { cneg_mod_256(ret, a, is_zero(flag) ^ 1, BLS12_381_r); } void blst_fr_to(vec256 ret, const vec256 a) -{ mul_mont_sparse_256(ret, a, BLS12_381_rRR, BLS12_381_r, r0); } +{ mul_mont_sparse_256(ret, BLS12_381_rRR, a, BLS12_381_r, r0); } void blst_fr_from(vec256 ret, const vec256 a) { from_mont_256(ret, a, BLS12_381_r, r0); } @@ -82,7 +82,7 @@ void blst_fr_from_scalar(vec256 ret, const pow256 a) } else { vec256 out; limbs_from_le_bytes(out, a, 32); - mul_mont_sparse_256(ret, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, out, BLS12_381_r, r0); vec_zero(out, sizeof(out)); } } @@ -151,14 +151,14 @@ void blst_sk_inverse(pow256 ret, const pow256 a) if (((size_t)a|(size_t)ret)%sizeof(limb_t) == 0 && is_endian.little) { limb_t *out = (limb_t *)ret; - mul_mont_sparse_256(out, (const limb_t *)a, BLS12_381_rRR, - BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, (const limb_t *)a, + BLS12_381_r, r0); reciprocal_fr(out, out); from_mont_256(out, out, BLS12_381_r, r0); } else { vec256 out; limbs_from_le_bytes(out, a, 32); - mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, out, BLS12_381_r, r0); reciprocal_fr(out, out); from_mont_256(out, out, BLS12_381_r, r0); le_bytes_from_limbs(ret, out, 32); @@ -472,7 +472,7 @@ void blst_fr_from_uint64(vec256 ret, const unsigned long long a[4]) } a = (const unsigned long long *)ret; } - mul_mont_sparse_256(ret, (const limb_t *)a, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, (const limb_t *)a, BLS12_381_r, r0); } void blst_uint64_from_fr(unsigned long long ret[4], const vec256 a) @@ -497,7 +497,7 @@ void blst_uint64_from_fr(unsigned long long ret[4], const vec256 a) int blst_scalar_from_le_bytes(pow256 out, const unsigned char *bytes, size_t n) { - size_t rem = (n - 1) % 32 + 1; + size_t rem = n ? ((n - 1) % 32 + 1) : 0; struct { vec256 out, digit; } t; limb_t ret; @@ -525,7 +525,7 @@ int blst_scalar_from_le_bytes(pow256 out, const unsigned char *bytes, size_t n) int blst_scalar_from_be_bytes(pow256 out, const unsigned char *bytes, size_t n) { - size_t rem = (n - 1) % 32 + 1; + size_t rem = n ? ((n - 1) % 32 + 1) : 0; struct { vec256 out, digit; } t; limb_t ret; @@ -550,6 +550,43 @@ int blst_scalar_from_be_bytes(pow256 out, const unsigned char *bytes, size_t n) return (int)(ret^1); } +void blst_fp_from_le_bytes(vec384 out, const unsigned char *bytes, size_t n) +{ + size_t rem = n ? ((n - 1) % 48 + 1) : 0; + vec384 digit; + + vec_zero(out, sizeof(vec384)); + + n -= rem; + limbs_from_le_bytes(out, bytes += n, rem); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + + while (n) { + limbs_from_le_bytes(digit, bytes -= 48, 48); + add_mod_384(out, out, digit, BLS12_381_P); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + n -= 48; + } +} + +void blst_fp_from_be_bytes(vec384 out, const unsigned char *bytes, size_t n) +{ + size_t rem = n ? ((n - 1) % 48 + 1) : 0; + vec384 digit; + + vec_zero(out, sizeof(vec384)); + + limbs_from_be_bytes(out, bytes, rem); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + + while (n -= rem) { + limbs_from_be_bytes(digit, bytes += rem, 48); + add_mod_384(out, out, digit, BLS12_381_P); + mul_mont_384(out, BLS12_381_RR, out, BLS12_381_P, p0); + rem = 48; + } +} + /* * Single-short SHA-256 hash function. */ @@ -573,7 +610,7 @@ void blst_scalar_from_hexascii(pow256 ret, const char *hex) void blst_fr_from_hexascii(vec256 ret, const char *hex) { limbs_from_hexascii(ret, sizeof(vec256), hex); - mul_mont_sparse_256(ret, ret, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(ret, BLS12_381_rRR, ret, BLS12_381_r, r0); } void blst_fp_from_hexascii(vec384 ret, const char *hex) diff --git a/blst_src/keygen.c b/blst_src/keygen.c index 9b62f16b..f904331d 100644 --- a/blst_src/keygen.c +++ b/blst_src/keygen.c @@ -33,7 +33,7 @@ static void HMAC_init(HMAC_SHA256_CTX *ctx, const void *K, size_t K_len) sha256_init(&ctx->ctx); sha256_update(&ctx->ctx, K, K_len); sha256_final(ctx->tail.c, &ctx->ctx); - } else { + } else if (K_len != 0) { sha256_bcopy(ctx->tail.c, K, K_len); } diff --git a/blst_src/multi_scalar.c b/blst_src/multi_scalar.c index 55ab8227..ce3f5184 100644 --- a/blst_src/multi_scalar.c +++ b/blst_src/multi_scalar.c @@ -7,9 +7,6 @@ #include "fields.h" #include "point.h" -/* - * Infinite point among inputs would be devastating. Shall we change it? - */ #define POINTS_TO_AFFINE_IMPL(prefix, ptype, bits, field) \ static void ptype##s_to_affine(ptype##_affine dst[], \ const ptype *const points[], size_t npoints) \ @@ -25,27 +22,36 @@ static void ptype##s_to_affine(ptype##_affine dst[], \ \ point = *points ? *points++ : point+1; \ acc = (vec##bits *)dst; \ - vec_copy(acc++, point->Z, sizeof(vec##bits)); \ - for (i = 1; i < delta; i++, acc++) \ - point = *points ? *points++ : point+1, \ - mul_##field(acc[0], acc[-1], point->Z); \ + vec_select(acc++, BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + for (i = 1; i < delta; i++, acc++) { \ + point = *points ? *points++ : point+1; \ + vec_select(acc[0], BLS12_381_Rx.p, point->Z, sizeof(vec##bits), \ + vec_is_zero(point->Z, sizeof(point->Z))); \ + mul_##field(acc[0], acc[0], acc[-1]); \ + } \ \ --acc; reciprocal_##field(acc[0], acc[0]); \ \ walkback = points-1, p = point, --delta, dst += delta; \ for (i = 0; i < delta; i++, acc--, dst--) { \ + bool_t is_inf = vec_is_zero(p->Z, sizeof(p->Z)); \ mul_##field(acc[-1], acc[-1], acc[0]); /* 1/Z */\ sqr_##field(ZZ, acc[-1]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[-1]); /* 1/Z^3 */\ - mul_##field(acc[-1], p->Z, acc[0]); \ + vec_select(acc[-1], BLS12_381_Rx.p, p->Z, sizeof(vec##bits), \ + is_inf); \ + mul_##field(acc[-1], acc[-1], acc[0]); \ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), is_inf); \ p = (p == *walkback) ? *--walkback : p-1; \ } \ sqr_##field(ZZ, acc[0]); /* 1/Z^2 */\ mul_##field(ZZZ, ZZ, acc[0]); /* 1/Z^3 */\ mul_##field(dst->X, p->X, ZZ); /* X = X'/Z^2 */\ mul_##field(dst->Y, p->Y, ZZZ); /* Y = Y'/Z^3 */\ + vec_czero(dst, sizeof(*dst), vec_is_zero(p->Z, sizeof(p->Z))); \ ++delta, dst += delta, npoints -= delta; \ } \ } \ @@ -74,18 +80,23 @@ POINTS_TO_AFFINE_IMPL(blst_p2, POINTonE2, 384x, fp2) #define SCRATCH_SZ(ptype) (sizeof(ptype)==sizeof(POINTonE1) ? 8192 : 4096) +/* The intermediate infinity points are encoded as [0, 0, 1]. */ + #define PRECOMPUTE_WBITS_IMPL(prefix, ptype, bits, field, one) \ -static void ptype##_precompute_row_wbits(ptype row[], size_t wbits, \ - const ptype##_affine *point) \ +static void ptype##_precompute_row(ptype row[], size_t n, \ + const ptype##_affine *point) \ { \ - size_t i, j, n = (size_t)1 << (wbits-1); \ + size_t i, j; \ + bool_t inf = vec_is_zero(point, sizeof(*point)); \ /* row[-1] is implicit infinity */\ vec_copy(&row[0], point, sizeof(*point)); /* row[0]=p*1 */\ vec_copy(&row[0].Z, one, sizeof(row[0].Z)); \ ptype##_double(&row[1], &row[0]); /* row[1]=p*(1+1) */\ + vec_select(&row[1].Z, one, &row[1].Z, sizeof(row[1].Z), inf); \ for (i = 2, j = 1; i < n; i += 2, j++) \ ptype##_add_affine(&row[i], &row[i-1], point), /* row[2]=p*(2+1) */\ - ptype##_double(&row[i+1], &row[j]); /* row[3]=p*(2+2) */\ + ptype##_double(&row[i+1], &row[j]), /* row[3]=p*(2+2) */\ + vec_select(&row[i+1].Z, one, &row[i+1].Z, sizeof(row[i+1].Z), inf); \ } /* row[4] ... */\ \ static void ptype##s_to_affine_row_wbits(ptype##_affine dst[], ptype src[], \ @@ -142,16 +153,44 @@ static void ptype##s_precompute_wbits(ptype##_affine table[], size_t wbits, \ rows = row = (ptype *)(&table[top]); \ for (i = 0; i < stride; i++, row += nwin) \ point = *points ? *points++ : point+1, \ - ptype##_precompute_row_wbits(row, wbits, point); \ + ptype##_precompute_row(row, nwin, point); \ ptype##s_to_affine_row_wbits(&table[top], rows, wbits, stride); \ top += stride << (wbits-1); \ npoints -= stride; \ } \ - rows = row = alloca(2*sizeof(ptype##_affine) * npoints * nwin); \ - for (i = 0; i < npoints; i++, row += nwin) \ - point = *points ? *points++ : point+1, \ - ptype##_precompute_row_wbits(row, wbits, point); \ - ptype##s_to_affine_row_wbits(&table[top], rows, wbits, npoints); \ + if ((i = 2*sizeof(ptype##_affine)*npoints*nwin) <= SCRATCH_LIMIT) { \ + rows = row = alloca(i); \ + for (i = 0; i < npoints; i++, row += nwin) \ + point = *points ? *points++ : point+1, \ + ptype##_precompute_row(row, nwin, point); \ + ptype##s_to_affine_row_wbits(&table[top], rows, wbits, npoints); \ + } else { \ + const ptype *pp[2]; \ +\ + stride = SCRATCH_LIMIT / sizeof(ptype); \ + stride -= stride % 2; \ + if (stride > nwin) stride = nwin; \ +\ + pp[0] = row = alloca(stride * sizeof(ptype)); \ + pp[1] = NULL; \ + for (i = 0; i < npoints; i++, top += nwin) { \ + size_t j, k, n; \ +\ + point = *points ? *points++ : point+1; \ + ptype##_precompute_row(row, stride, point); \ + ptype##s_to_affine(&table[top], pp, stride); \ + for (j = stride; j < nwin; j += stride) { \ + n = (j+stride) <= nwin ? stride : nwin-j; \ + for (k = 0; k < n-1; k++) \ + ptype##_add_affine(&row[k], &row[stride-1], &table[top+k]); \ + if (j == stride) \ + ptype##_double(&row[k], &row[stride-1]); \ + else \ + ptype##_add_affine(&row[k], &row[stride-1], &table[top+k]); \ + ptype##s_to_affine(&table[top+j], pp, n); \ + } \ + } \ + } \ } \ \ size_t prefix##s_mult_wbits_precompute_sizeof(size_t wbits, size_t npoints) \ @@ -202,7 +241,7 @@ static void ptype##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ \ nbits -= window; \ z = is_zero(nbits); \ - wval = (get_wval_limb(scalar, nbits - (z^1), wbits + (z^1)) << z) & wmask; \ + wval = (get_wval_limb(scalar, nbits - (z^1), window + (z^1)) << z) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[0], row, wbits, wval); \ row += nwin; \ @@ -232,7 +271,7 @@ static void ptype##s_mult_wbits(ptype *ret, const ptype##_affine table[], \ if (j == scratch_sz) \ ptype##s_accumulate(ret, scratch, j), j = 0; \ scalar = *scalar_s ? *scalar_s++ : scalar+nbytes; \ - wval = (get_wval_limb(scalar, 0, wbits) << 1) & wmask; \ + wval = (get_wval_limb(scalar, 0, window) << 1) & wmask; \ wval = booth_encode(wval, wbits); \ ptype##_gather_booth_wbits(&scratch[j], row, wbits, wval); \ } \ @@ -267,7 +306,14 @@ static size_t pippenger_window_size(size_t npoints) for (wbits=0; npoints>>=1; wbits++) ; - return wbits>12 ? wbits-3 : (wbits>4 ? wbits-2 : (wbits ? 2 : 1)); + if (wbits > 12) + return wbits - 3; + else if (wbits > 8) + return wbits - 2; + else if (wbits > 4) + return wbits - 1; + + return wbits ? 2 : 1; } #define DECLARE_PRIVATE_POINTXYZZ(ptype, bits) \ @@ -402,10 +448,11 @@ void prefix##s_mult_pippenger(ptype *ret, \ { \ if (npoints == 1) { \ prefix##_from_affine(ret, points[0]); \ - prefix##_mult(ret, ret, scalars[0], nbits); \ + ptype##_mult_w5(ret, ret, scalars[0], nbits); \ return; \ } \ - if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT) { \ + if ((npoints * sizeof(ptype##_affine) * 8 * 3) <= SCRATCH_LIMIT && \ + npoints < 32) { \ ptype##_affine *table = alloca(npoints * sizeof(ptype##_affine) * 8); \ ptype##s_precompute_wbits(table, 4, points, npoints); \ ptype##s_mult_wbits(ret, table, 4, npoints, scalars, nbits, NULL); \ diff --git a/blst_src/no_asm.h b/blst_src/no_asm.h index be7bf47e..80bbba57 100644 --- a/blst_src/no_asm.h +++ b/blst_src/no_asm.h @@ -635,58 +635,6 @@ void sqr_n_mul_mont_383(vec384 ret, const vec384 a, size_t count, mul_mont_n(ret, ret, b, p, n0, NLIMBS(384)); } -void sqr_mont_382x(vec384x ret, const vec384x a, - const vec384 p, limb_t n0) -{ - llimb_t limbx; - limb_t mask, carry, borrow; - size_t i; - vec384 t0, t1; - - /* "add_mod_n(t0, a[0], a[1], p, NLIMBS(384));" */ - for (carry=0, i=0; i> LIMB_T_BITS); - } - - /* "sub_mod_n(t1, a[0], a[1], p, NLIMBS(384));" */ - for (borrow=0, i=0; i> LIMB_T_BITS) & 1; - } - mask = 0 - borrow; - launder(mask); - - /* "mul_mont_n(ret[1], a[0], a[1], p, n0, NLIMBS(384));" */ - mul_mont_nonred_n(ret[1], a[0], a[1], p, n0, NLIMBS(384)); - - /* "add_mod_n(ret[1], ret[1], ret[1], p, NLIMBS(384));" */ - for (carry=0, i=0; i>(LIMB_T_BITS-1); - } - - /* "mul_mont_n(ret[0], t0, t1, p, n0, NLIMBS(384));" */ - mul_mont_nonred_n(ret[0], t0, t1, p, n0, NLIMBS(384)); - - /* account for t1's sign... */ - for (borrow=0, i=0; i> LIMB_T_BITS) & 1; - } - mask = 0 - borrow; - launder(mask); - for (carry=0, i=0; i> LIMB_T_BITS); - } -} - #if defined(__GNUC__) || defined(__clang__) # define MSB(x) ({ limb_t ret = (x) >> (LIMB_T_BITS-1); launder(ret); ret; }) #else diff --git a/blst_src/recip.c b/blst_src/recip.c index e0c70063..85f00228 100644 --- a/blst_src/recip.c +++ b/blst_src/recip.c @@ -62,20 +62,11 @@ static void reciprocal_fp(vec384 out, const vec384 inp) TO_LIMB_T(0x39869507b587b120), TO_LIMB_T(0x23ba5c279c2895fb), TO_LIMB_T(0x58dd3db21a5d66bb), TO_LIMB_T(0xd0088f51cbff34d2) }; -#ifdef __BLST_NO_ASM__ -# define RRx4 BLS12_381_RR -#else - static const vec384 RRx4 = { /* (4<<768)%P */ - TO_LIMB_T(0x5f7e7cd070d107c2), TO_LIMB_T(0xec839a9ac49c13c8), - TO_LIMB_T(0x6933786f44f4ef0b), TO_LIMB_T(0xd6bf8b9c676be983), - TO_LIMB_T(0xd3adaaaa4dcefb06), TO_LIMB_T(0x12601bc1d82bc175) - }; -#endif union { vec768 x; vec384 r[2]; } temp; - ct_inverse_mod_383(temp.x, inp, BLS12_381_P, Px8); + ct_inverse_mod_384(temp.x, inp, BLS12_381_P, Px8); redc_mont_384(temp.r[0], temp.x, BLS12_381_P, p0); - mul_mont_384(temp.r[0], temp.r[0], RRx4, BLS12_381_P, p0); + mul_mont_384(temp.r[0], temp.r[0], BLS12_381_RR, BLS12_381_P, p0); #ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION /* sign goes straight to flt_reciprocal */ @@ -88,7 +79,6 @@ static void reciprocal_fp(vec384 out, const vec384 inp) #else vec_copy(out, temp.r[0], sizeof(vec384)); #endif -#undef RRx4 } void blst_fp_inverse(vec384 out, const vec384 inp) @@ -129,7 +119,7 @@ static void reciprocal_fr(vec256 out, const vec256 inp) ct_inverse_mod_256(temp, inp, BLS12_381_r, rx2); redc_mont_256(out, temp, BLS12_381_r, r0); - mul_mont_sparse_256(out, out, BLS12_381_rRR, BLS12_381_r, r0); + mul_mont_sparse_256(out, BLS12_381_rRR, out, BLS12_381_r, r0); } void blst_fr_inverse(vec256 out, const vec256 inp) diff --git a/blst_src/vect.h b/blst_src/vect.h index 19640b11..6b48d633 100644 --- a/blst_src/vect.h +++ b/blst_src/vect.h @@ -85,9 +85,7 @@ typedef limb_t bool_t; # define from_mont_384 fromx_mont_384 # define sgn0_pty_mont_384 sgn0x_pty_mont_384 # define sgn0_pty_mont_384x sgn0x_pty_mont_384x -# define ct_inverse_mod_383 ctx_inverse_mod_383 -#elif defined(__BLST_NO_ASM__) -# define ct_inverse_mod_383 ct_inverse_mod_384 +# define ct_inverse_mod_384 ctx_inverse_mod_384 #endif void mul_mont_sparse_256(vec256 ret, const vec256 a, const vec256 b, @@ -137,7 +135,7 @@ void cneg_mod_384(vec384 ret, const vec384 a, bool_t flag, const vec384 p); void lshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void rshift_mod_384(vec384 ret, const vec384 a, size_t count, const vec384 p); void div_by_2_mod_384(vec384 ret, const vec384 a, const vec384 p); -void ct_inverse_mod_383(vec768 ret, const vec384 inp, const vec384 mod, +void ct_inverse_mod_384(vec768 ret, const vec384 inp, const vec384 mod, const vec384 modx); void ct_inverse_mod_256(vec512 ret, const vec256 inp, const vec256 mod, const vec256 modx); @@ -146,7 +144,6 @@ bool_t ct_is_square_mod_384(const vec384 inp, const vec384 mod); #if defined(__ADX__) /* e.g. -march=broadwell */ && !defined(__BLST_PORTABLE__) # define mul_mont_384x mulx_mont_384x # define sqr_mont_384x sqrx_mont_384x -# define sqr_mont_382x sqrx_mont_382x # define mul_382x mulx_382x # define sqr_382x sqrx_382x #endif @@ -154,7 +151,6 @@ bool_t ct_is_square_mod_384(const vec384 inp, const vec384 mod); void mul_mont_384x(vec384x ret, const vec384x a, const vec384x b, const vec384 p, limb_t n0); void sqr_mont_384x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); -void sqr_mont_382x(vec384x ret, const vec384x a, const vec384 p, limb_t n0); void mul_382x(vec768 ret[2], const vec384x a, const vec384x b, const vec384 p); void sqr_382x(vec768 ret[2], const vec384x a, const vec384 p); @@ -407,7 +403,7 @@ static inline void vec_czero(void *ret, size_t num, bool_t cbit) #if defined(__INTEL_COMPILER) # pragma warning(disable:167) # pragma warning(disable:556) -#elif defined(__GNUC__) && !defined(__clang__) +#elif defined(__GNUC__) && !defined(__clang__) && (__STDC_VERSION__-0) < 202311 # pragma GCC diagnostic ignored "-Wpedantic" #elif defined(_MSC_VER) # pragma warning(disable: 4127 4189) diff --git a/internal/blst/blst.go b/internal/blst/blst.go index 6fe6b318..a40fdbd6 100644 --- a/internal/blst/blst.go +++ b/internal/blst/blst.go @@ -301,8 +301,12 @@ func KeyGenV5(ikm []byte, salt []byte, optional ...[]byte) *SecretKey { if len(optional) > 0 { info = optional[0] } + saltLen := len(salt) + if saltLen == 0 { + salt = []byte{0} + } C.blst_keygen_v5(&sk.cgo, (*C.byte)(&ikm[0]), C.size_t(len(ikm)), - (*C.byte)(&salt[0]), C.size_t(len(salt)), + (*C.byte)(&salt[0]), C.size_t(saltLen), ptrOrNil(info), C.size_t(len(info))) // Postponing secret key zeroing till garbage collection can be too // late to be effective, but every little bit helps... @@ -370,14 +374,6 @@ func PairingAsFp12(ctx Pairing) *Fp12 { return &pt } -func (pt *Fp12) asPtr() *C.blst_fp12 { - if pt != nil { - return &pt.cgo - } - - return nil -} - func Fp12One() Fp12 { return cgo_fp12One } @@ -472,6 +468,14 @@ func (pt1 *Fp12) Equals(pt2 *Fp12) bool { return *pt1 == *pt2 } +func (pt *Fp12) asPtr() *C.blst_fp12 { + if pt != nil { + return &pt.cgo + } + + return nil +} + func ptrOrNil(bytes []byte) *C.byte { var ptr *C.byte if len(bytes) > 0 { @@ -488,14 +492,6 @@ func ptrOrNil(bytes []byte) *C.byte { // PublicKey // -func (pt *P1Affine) asPtr() *C.blst_p1_affine { - if pt != nil { - return &pt.cgo - } - - return nil -} - func (pk *P1Affine) From(s *Scalar) *P1Affine { C.blst_sk_to_pk2_in_g1(nil, &pk.cgo, &s.cgo) return pk @@ -604,7 +600,7 @@ func (sig *P2Affine) AggregateVerify(sigGroupcheck bool, // Aggregate verify with compressed signature and public keys // Uses a dummy signature to get the correct type -func (_ *P2Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, +func (*P2Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, pks [][]byte, pksVerify bool, msgs []Message, dst []byte, optional ...bool) bool { // useHash bool, usePksAsAugs bool @@ -810,7 +806,7 @@ func (sig *P2Affine) FastAggregateVerify(sigGroupcheck bool, return sig.Verify(sigGroupcheck, pkAff, false, msg, dst, optional...) } -func (_ *P2Affine) MultipleAggregateVerify(sigs []*P2Affine, +func (*P2Affine) MultipleAggregateVerify(sigs []*P2Affine, sigsGroupcheck bool, pks []*P1Affine, pksVerify bool, msgs []Message, dst []byte, randFn func(*Scalar), randBits int, optional ...interface{}) bool { // useHash @@ -1096,14 +1092,6 @@ func (agg *P2Aggregate) coreAggregate(getter aggGetterP2, groupcheck bool, // PublicKey // -func (pt *P2Affine) asPtr() *C.blst_p2_affine { - if pt != nil { - return &pt.cgo - } - - return nil -} - func (pk *P2Affine) From(s *Scalar) *P2Affine { C.blst_sk_to_pk2_in_g2(nil, &pk.cgo, &s.cgo) return pk @@ -1212,7 +1200,7 @@ func (sig *P1Affine) AggregateVerify(sigGroupcheck bool, // Aggregate verify with compressed signature and public keys // Uses a dummy signature to get the correct type -func (_ *P1Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, +func (*P1Affine) AggregateVerifyCompressed(sig []byte, sigGroupcheck bool, pks [][]byte, pksVerify bool, msgs []Message, dst []byte, optional ...bool) bool { // useHash bool, usePksAsAugs bool @@ -1418,7 +1406,7 @@ func (sig *P1Affine) FastAggregateVerify(sigGroupcheck bool, return sig.Verify(sigGroupcheck, pkAff, false, msg, dst, optional...) } -func (_ *P1Affine) MultipleAggregateVerify(sigs []*P1Affine, +func (*P1Affine) MultipleAggregateVerify(sigs []*P1Affine, sigsGroupcheck bool, pks []*P2Affine, pksVerify bool, msgs []Message, dst []byte, randFn func(*Scalar), randBits int, optional ...interface{}) bool { // useHash @@ -1721,6 +1709,10 @@ func PairingMulNAggregatePkInG1(ctx Pairing, PK *P1Affine, pkValidate bool, aug = optional[0] } + if randBits > 256 { + panic("scalar length mismatch") + } + r := C.blst_pairing_chk_n_mul_n_aggr_pk_in_g1(&ctx[0], PK.asPtr(), C.bool(pkValidate), sig.asPtr(), C.bool(sigGroupcheck), @@ -1771,7 +1763,7 @@ func (p1 *P1Affine) InG1() bool { return bool(C.blst_p1_affine_in_g1(&p1.cgo)) } -func (_ *P1Affine) BatchUncompress(in [][]byte) []*P1Affine { +func (*P1Affine) BatchUncompress(in [][]byte) []*P1Affine { // Allocate space for all of the resulting points. Later we'll save pointers // and return those so that the result could be used in other functions, // such as MultipleAggregateVerify. @@ -2172,6 +2164,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { case P1Affines: pointsBySlice[0] = &val[0].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } scalarsBySlice := [2]*C.byte{nil, nil} @@ -2191,6 +2184,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: p_scalars = &scalars[0] + default: // type is already vetted } var ret P1 @@ -2206,18 +2200,19 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { return &ret } - if npoints < 32 { + if npoints < 32 || npoints < numThreads { if numThreads > npoints { numThreads = npoints } + acc := make([]P1, numThreads) + curItem := uint32(0) - msgs := make(chan P1, numThreads) + var wg sync.WaitGroup + wg.Add(numThreads) for tid := 0; tid < numThreads; tid++ { - go func() { - var acc P1 - + go func(acc *P1) { for { workItem := int(atomic.AddUint32(&curItem, 1) - 1) if workItem >= npoints { @@ -2232,6 +2227,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { point = &val[workItem] case P1Affines: point = &val[workItem] + default: // type is already vetted } var scalar *C.byte @@ -2248,20 +2244,22 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: scalar = scalars[workItem] + default: // type is already vetted } C.go_p1_mult_n_acc(&acc.cgo, &point.cgo.x, true, scalar, C.size_t(nbits)) } - msgs <- acc - }() + wg.Done() + }(&acc[tid]) } - ret := <-msgs + wg.Wait() + + ret := acc[0] for tid := 1; tid < numThreads; tid++ { - point := <-msgs - C.blst_p1_add_or_double(&ret.cgo, &ret.cgo, &point.cgo) + C.blst_p1_add_or_double(&ret.cgo, &ret.cgo, &acc[tid].cgo) } for i := range scalars { @@ -2338,6 +2336,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { case P1Affines: pointsBySlice[0] = &val[x].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } var p_scalars **C.byte @@ -2356,6 +2355,7 @@ func P1AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P1 { } case []*Scalar: p_scalars = &scalars[x] + default: // type is already vetted } C.blst_p1s_tile_pippenger(&grid[workItem].point.cgo, @@ -2528,6 +2528,10 @@ func PairingMulNAggregatePkInG2(ctx Pairing, PK *P2Affine, pkValidate bool, aug = optional[0] } + if randBits > 256 { + panic("scalar length mismatch") + } + r := C.blst_pairing_chk_n_mul_n_aggr_pk_in_g2(&ctx[0], PK.asPtr(), C.bool(pkValidate), sig.asPtr(), C.bool(sigGroupcheck), @@ -2578,7 +2582,7 @@ func (p2 *P2Affine) InG2() bool { return bool(C.blst_p2_affine_in_g2(&p2.cgo)) } -func (_ *P2Affine) BatchUncompress(in [][]byte) []*P2Affine { +func (*P2Affine) BatchUncompress(in [][]byte) []*P2Affine { // Allocate space for all of the resulting points. Later we'll save pointers // and return those so that the result could be used in other functions, // such as MultipleAggregateVerify. @@ -2979,6 +2983,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { case P2Affines: pointsBySlice[0] = &val[0].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } scalarsBySlice := [2]*C.byte{nil, nil} @@ -2998,6 +3003,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: p_scalars = &scalars[0] + default: // type is already vetted } var ret P2 @@ -3013,18 +3019,19 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { return &ret } - if npoints < 32 { + if npoints < 32 || npoints < numThreads { if numThreads > npoints { numThreads = npoints } + acc := make([]P2, numThreads) + curItem := uint32(0) - msgs := make(chan P2, numThreads) + var wg sync.WaitGroup + wg.Add(numThreads) for tid := 0; tid < numThreads; tid++ { - go func() { - var acc P2 - + go func(acc *P2) { for { workItem := int(atomic.AddUint32(&curItem, 1) - 1) if workItem >= npoints { @@ -3039,6 +3046,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { point = &val[workItem] case P2Affines: point = &val[workItem] + default: // type is already vetted } var scalar *C.byte @@ -3055,20 +3063,22 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: scalar = scalars[workItem] + default: // type is already vetted } C.go_p2_mult_n_acc(&acc.cgo, &point.cgo.x, true, scalar, C.size_t(nbits)) } - msgs <- acc - }() + wg.Done() + }(&acc[tid]) } - ret := <-msgs + wg.Wait() + + ret := acc[0] for tid := 1; tid < numThreads; tid++ { - point := <-msgs - C.blst_p2_add_or_double(&ret.cgo, &ret.cgo, &point.cgo) + C.blst_p2_add_or_double(&ret.cgo, &ret.cgo, &acc[tid].cgo) } for i := range scalars { @@ -3145,6 +3155,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { case P2Affines: pointsBySlice[0] = &val[x].cgo p_points = &pointsBySlice[0] + default: // type is already vetted } var p_scalars **C.byte @@ -3163,6 +3174,7 @@ func P2AffinesMult(pointsIf interface{}, scalarsIf interface{}, nbits int) *P2 { } case []*Scalar: p_scalars = &scalars[x] + default: // type is already vetted } C.blst_p2s_tile_pippenger(&grid[workItem].point.cgo, @@ -3547,6 +3559,14 @@ func (e1 *P1Affine) Equals(e2 *P1Affine) bool { return bool(C.blst_p1_affine_is_equal(&e1.cgo, &e2.cgo)) } +func (pt *P1Affine) asPtr() *C.blst_p1_affine { + if pt != nil { + return &pt.cgo + } + + return nil +} + func (e1 *P1) Equals(e2 *P1) bool { return bool(C.blst_p1_is_equal(&e1.cgo, &e2.cgo)) } @@ -3555,6 +3575,14 @@ func (e1 *P2Affine) Equals(e2 *P2Affine) bool { return bool(C.blst_p2_affine_is_equal(&e1.cgo, &e2.cgo)) } +func (pt *P2Affine) asPtr() *C.blst_p2_affine { + if pt != nil { + return &pt.cgo + } + + return nil +} + func (e1 *P2) Equals(e2 *P2) bool { return bool(C.blst_p2_is_equal(&e1.cgo, &e2.cgo)) } @@ -3585,15 +3613,18 @@ func breakdown(nbits, window, ncpus int) (nx int, ny int, wnd int) { wnd = window } } - } else { + } else if window > 3 { nx = 2 wnd = window - 2 - for (nbits/wnd+1)*nx < ncpus { + for wnd > 1 && (nbits/wnd+1)*nx < ncpus { nx += 1 wnd = window - bits.Len(3*uint(nx)/2) } nx -= 1 wnd = window - bits.Len(3*uint(nx)/2) + } else { + nx = 1 + wnd = window } ny = nbits/wnd + 1 wnd = nbits/ny + 1 diff --git a/internal/blst/blst.h b/internal/blst/blst.h index 8d90abf0..e234b395 100644 --- a/internal/blst/blst.h +++ b/internal/blst/blst.h @@ -26,12 +26,12 @@ extern "C" { #elif !defined(__STDC_VERSION__) || __STDC_VERSION__<202311 # if defined(__BLST_CGO__) typedef _Bool bool; /* it's assumed that cgo calls modern enough compiler */ +# elif defined(__BLST_RUST_BINDGEN__) || defined(__BLST_ZIG__) +# define bool _Bool +# elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 +# include # elif !defined(bool) -# if defined(__STDC_VERSION__) && __STDC_VERSION__>=199901 -# define bool _Bool -# else -# define bool int -# endif +# define bool int # define __blst_h_bool__ # endif #endif diff --git a/internal/blst/blst_aux.h b/internal/blst/blst_aux.h index 3de0850e..394a94dd 100644 --- a/internal/blst/blst_aux.h +++ b/internal/blst/blst_aux.h @@ -110,6 +110,9 @@ size_t blst_p2_sizeof(void); size_t blst_p2_affine_sizeof(void); size_t blst_fp12_sizeof(void); +void blst_fp_from_le_bytes(blst_fp *ret, const byte *in, size_t len); +void blst_fp_from_be_bytes(blst_fp *ret, const byte *in, size_t len); + /* * Single-shot SHA-256 hash function. */ From c574d6b7b7b61b29033faa08177a1bbd40a7bcf8 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 3 Aug 2026 19:06:27 -0500 Subject: [PATCH 08/21] ecdsa improvements --- ecdsa.go | 11 ++++------- ecdsa_p256.go | 2 +- ecdsa_secp256k1.go | 22 ++++++++++++++++------ sign.go | 6 +++++- sign_test_utils.go | 12 ++++++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index 9e4e715b..a0b71211 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -62,10 +62,10 @@ func (a *ecdsaContext) checkAlgoAndComputeHash(msg []byte, hasher hash.Hasher) ( } // check hasher's size is at least the curve order in bytes - nLen := bitsToBytes((a.curveN).BitLen()) - if hasher.Size() < nLen { + nLen := (a.curveN).BitLen() + if (hasher.Size() << 3) < nLen { return nil, invalidHasherSizeErrorf( - "hasher's size should be at least %d, got %d", nLen, hasher.Size()) + "hasher's bit-size should be at least %d, got %d", nLen, hasher.Size()<<3) } h := hasher.ComputeHash(msg) @@ -118,7 +118,7 @@ func (a *ecdsaContext) mapToPrivateKey(seed []byte) (PrivateKey, error) { // privateKey returns an ECDSA private key using the // input scalar. - +// // Input scalar d is assumed to be satisfy 0 < d < n before calling this function. // // The function returns: @@ -212,8 +212,6 @@ func (a *ecdsaContext) decodePrivateKey(der []byte) (PrivateKey, error) { // Error Returns: // - invalidInputsError if the input is not a valid serialization of a public key on the given curve. func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { - // all the curves supported for now have a cofactor equal to 1, - // so that checking the point is on curve is enough to make sure it is on the correct subgroup switch a.algo { case ECDSAP256: return publicKeyECDSAP256(a, input) @@ -222,7 +220,6 @@ func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { default: return nil, invalidInputsErrorf("curve is not supported") } - } func (a *ecdsaContext) decodePublicKey(der []byte) (PublicKey, error) { diff --git a/ecdsa_p256.go b/ecdsa_p256.go index d50d76dc..0e5159ba 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -85,7 +85,7 @@ func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error sk := &prKeyECDSAP256{ prKeyCommonECDSA: &prKeyCommonECDSA{a}, goPrKey: internalSK, - pubKey: nil, // public key is not constructed + pubKey: nil, // public key is not constructed yet } return sk, nil } diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index f23fdded..669eab99 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -93,7 +93,7 @@ func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256 sk := &prKeyECDSASecp256k1{ prKeyCommonECDSA: &prKeyCommonECDSA{a}, dBytes: dBytes, - pubKey: nil, // public key is not constructed + pubKey: nil, // public key is not constructed yet } return sk } @@ -115,8 +115,8 @@ func (sk *prKeyECDSASecp256k1) Sign(msg []byte, hasher hash.Hasher) (Signature, if err != nil { return nil, err } - // truncate the hash to the curve order size, as specified in FIPS 186-4 section 6.4 - // and as required by the secp256k1 package signing function + // truncate the hash to the curve order size, as specified in FIPS 186-4 section 6.4 (nLenSecp256k1 here is a multiple of 8 bits). + // Moreover, the secp256k1 package requires the message hash to equal nLenSecp256k1 hash = hash[:nLenSecp256k1] signature, err := secp256k1.Sign(hash, sk.dBytes) if err != nil { @@ -133,8 +133,16 @@ func (sk *prKeyECDSASecp256k1) String() string { func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp256k1, error) { pLen := bitsToBytes(a.curveP.BitLen()) + + if len(XYBytes) != 2*pLen { + return nil, invalidInputsErrorf("input has incorrect %s key size, got %d, expects %d", + a.algo, len(XYBytes), 2*pLen) + } + x, y := readTwoBigInts(XYBytes, pLen) + // `IsOnCurve` is not deprecated in btcec's type `KoblitzCurve` + // `IsOnCurve` includes checks for x

= len(names) { + return "UNKNOWN" + } + return names[f] } // Signature is a generic type, regardless of the signature scheme diff --git a/sign_test_utils.go b/sign_test_utils.go index b0f2a7ae..ebbd5150 100644 --- a/sign_test_utils.go +++ b/sign_test_utils.go @@ -243,6 +243,12 @@ func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { assert.True(t, IsInvalidInputsError(err)) assert.Nil(t, sk) + bytes = make([]byte, skLens[salg]-1) + sk, err = DecodePrivateKey(salg, bytes) + require.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + assert.Nil(t, sk) + // public key pkLens := make(map[SigningAlgorithm]int) pkLens[ECDSAP256] = PubKeyLenECDSAP256 @@ -254,6 +260,12 @@ func testEncodeDecode(t *testing.T, salg SigningAlgorithm) { require.Error(t, err) assert.True(t, IsInvalidInputsError(err)) assert.Nil(t, pk) + + bytes = make([]byte, pkLens[salg]-1) + pk, err = DecodePublicKey(salg, bytes) + require.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + assert.Nil(t, pk) }) }) } From 2d4a72c2909c0d68ec14bc890ee8649f91f7ddc3 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Tue, 4 Aug 2026 15:05:54 -0500 Subject: [PATCH 09/21] address malleability breaking change --- ecdsa.go | 40 +++++++++++++++++++++++++ ecdsa_p256.go | 13 +++++--- ecdsa_secp256k1.go | 22 +++++++++----- ecdsa_test.go | 74 ++++++++++++++++++++++++++++++++++++++++++++-- go.mod | 2 -- go.sum | 5 ---- 6 files changed, 135 insertions(+), 21 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index a0b71211..d38e7c33 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -41,6 +41,8 @@ type ecdsaContext struct { curveP *big.Int // curve order curveN *big.Int + // curve order minus 1 divided by 2 (used for signature malleability annalysis) + curveNdiv2 *big.Int } const ecEncodingUncompressed = 0x4 @@ -100,6 +102,7 @@ func (a *ecdsaContext) signatureFormatCheck(sig Signature) bool { } var one = new(big.Int).SetInt64(1) +var two = new(big.Int).SetInt64(2) // mapToPrivateKey simply maps the input seed to an ECDSA private key // The private scalar `d` satisfies 0 < d < n. @@ -317,3 +320,40 @@ func readTwoBigInts(input []byte, size int) (*big.Int, *big.Int) { b := new(big.Int).SetBytes(input[size : 2*size]) return a, b } + +// isLowS returns true if the signature's S is in the lower range (S <= (n-1)/2). +func (a *ecdsaContext) isLowS(s *big.Int) bool { + return a.curveNdiv2.Cmp(s) >= 0 +} + +// signatureNormalizeS returns a new signature with S normalized to low S. +// This is needed when the signature verification requires low S to avoid signature malleability, while the package allows high S signatures to be accepted. +func (a *ecdsaContext) signatureNormalizeS(sig []byte) []byte { + // read S + nLen := bitsToBytes(a.curveN.BitLen()) + s := new(big.Int).SetBytes(sig[nLen:]) + if a.isLowS(s) { + return sig // no need to flip S + } + // compute N-S + sComplement := new(big.Int).Sub(a.curveN, s) + // write it into a new signature + newSig := make([]byte, len(sig)) + copy(newSig, sig[:nLen]) // copy R + sComplement.FillBytes(newSig[nLen:]) // write S complement + return newSig +} + +// Test function only to flip S in a signature. It is used for testing signature malleability +func (a *ecdsaContext) signatureFlipS(sig []byte) []byte { + // read S + nLen := bitsToBytes(a.curveN.BitLen()) + s := new(big.Int).SetBytes(sig[nLen:]) + // compute N-S + sComplement := new(big.Int).Sub(a.curveN, s) + // write it into a new signature + newSig := make([]byte, len(sig)) + copy(newSig, sig[:nLen]) // copy R + sComplement.FillBytes(newSig[nLen:]) // write S complement + return newSig +} diff --git a/ecdsa_p256.go b/ecdsa_p256.go index 0e5159ba..87f7b41f 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -23,6 +23,7 @@ import ( "crypto/elliptic" "crypto/rand" "fmt" + "math/big" "github.com/onflow/crypto/hash" ) @@ -35,7 +36,7 @@ const ( pLenP256 = 32 ) -var ( +const ( // NIST P256 SignatureLenECDSAP256 = 2 * nLenP256 PrKeyLenECDSAP256 = nLenP256 @@ -48,10 +49,14 @@ var p256Instance *ecdsaContext func initECDSAP256() { curve := elliptic.P256() + n := curve.Params().N + nMinus1 := new(big.Int).Sub(n, one) + p256Instance = &(ecdsaContext{ - curveP: curve.Params().P, - curveN: curve.Params().N, - algo: ECDSAP256, + curveP: curve.Params().P, + curveN: n, + curveNdiv2: new(big.Int).Div(nMinus1, two), // (N-1)/2 + algo: ECDSAP256, }) } diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index 669eab99..a6478bad 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -38,9 +38,11 @@ const ( nLenSecp256k1 = 32 pLenSecp256k1 = 32 + + secp256k1Ndiv2Hex = "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0" ) -var ( +const ( // SECG secp256k1 SignatureLenECDSASecp256k1 = 2 * nLenSecp256k1 PrKeyLenECDSASecp256k1 = nLenSecp256k1 @@ -60,10 +62,15 @@ func initECDSASecp256k1() { if !ok { panic("failed to initialize ECDSA with secp256k1 curve") } + curveNdiv2, ok := new(big.Int).SetString(secp256k1Ndiv2Hex, 16) + if !ok { + panic("failed to initialize ECDSA with secp256k1 curve") + } secp256k1Instance = &(ecdsaContext{ - curveP: curveP, - curveN: curveN, - algo: ECDSASecp256k1, + curveP: curveP, + curveN: curveN, + curveNdiv2: curveNdiv2, + algo: ECDSASecp256k1, }) } @@ -141,7 +148,6 @@ func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp2 x, y := readTwoBigInts(XYBytes, pLen) - // `IsOnCurve` is not deprecated in btcec's type `KoblitzCurve` // `IsOnCurve` includes checks for x

Date: Tue, 4 Aug 2026 15:11:24 -0500 Subject: [PATCH 10/21] cleanup --- ecdsa_secp256k1.go | 4 +++- ecdsa_test.go | 28 ---------------------------- 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index a6478bad..de8811c0 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -202,7 +202,9 @@ func (pk *pubKeyECDSASecp256k1) Verify(sig Signature, msg []byte, hasher hash.Ha if len(sig) != 2*nLenSecp256k1 { return false, nil } - // normalize the signature to low S. This is required because the secp256k1 package does not accept high S signatures. + // normalize the signature to low S. + // This is required because the secp256k1 package does not accept high S signatures while the package allows them. + // Rejecting high S signatures would be a breaking change with prior versions. newSig := secp256k1Instance.signatureNormalizeS(sig) // truncate the hash to the curve order size, as specified in FIPS 186-4 section 6.4 (nLenSecp256k1 here is a multiple of 8 bits). diff --git a/ecdsa_test.go b/ecdsa_test.go index 41f5ca06..5c26f49c 100644 --- a/ecdsa_test.go +++ b/ecdsa_test.go @@ -25,7 +25,6 @@ import ( crand "crypto/rand" - "github.com/ethereum/go-ethereum/crypto/secp256k1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -346,33 +345,6 @@ func TestECDSASignatureFormatCheck(t *testing.T) { } } -func TestEllipticUnmarshalSecp256k1(t *testing.T) { - testVectors := []string{ - "028b10bf56476bf7da39a3286e29df389177a2fa0fca2d73348ff78887515d8da1", // IsOnCurve for elliptic returns false - "03d39427f07f680d202fe8504306eb29041aceaf4b628c2c69b0ec248155443166", // odd, IsOnCurve for elliptic returns false - "0267d1942a6cbe4daec242ea7e01c6cdb82dadb6e7077092deb55c845bf851433e", // arith of sqrt in elliptic doesn't match secp256k1 - "0345d45eda6d087918b041453a96303b78c478dce89a4ae9b3c933a018888c5e06", // odd, arith of sqrt in elliptic doesn't match secp256k1 - } - - for _, testVector := range testVectors { - // get the compressed bytes - publicBytes, err := hex.DecodeString(testVector) - require.NoError(t, err) - - // decompress, check that those are perfectly valid Secp256k1 public keys - retrieved, err := DecodePublicKeyCompressed(ECDSASecp256k1, publicBytes) - require.NoError(t, err) - - // check the compression is canonical by re-compressing to the same bytes - require.Equal(t, retrieved.EncodeCompressed(), publicBytes) - - // check that elliptic fails at decompressing them - x, y := secp256k1.DecompressPubkey(publicBytes) - require.Nil(t, x) - require.Nil(t, y) - } -} - func BenchmarkECDSADecode(b *testing.B) { // random message seed := make([]byte, 50) From 7390c14a572cc852a7d8247eb3eb3c9a7d48b09f Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 5 Aug 2026 13:25:05 -0500 Subject: [PATCH 11/21] check S range before complementing --- ecdsa.go | 32 +++++++++---- ecdsa_secp256k1.go | 5 +- ecdsa_test.go | 115 +++++++++++++++++++++++++++------------------ 3 files changed, 94 insertions(+), 58 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index d38e7c33..e7cf8cb0 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -321,27 +321,39 @@ func readTwoBigInts(input []byte, size int) (*big.Int, *big.Int) { return a, b } -// isLowS returns true if the signature's S is in the lower range (S <= (n-1)/2). +// isLowS returns true if the signature's S is in the lower range (S <= (n-1)/2) func (a *ecdsaContext) isLowS(s *big.Int) bool { return a.curveNdiv2.Cmp(s) >= 0 } -// signatureNormalizeS returns a new signature with S normalized to low S. -// This is needed when the signature verification requires low S to avoid signature malleability, while the package allows high S signatures to be accepted. -func (a *ecdsaContext) signatureNormalizeS(sig []byte) []byte { +// signatureNormalizeS returns a signature with S normalized to low S. +// (same slice is returned if S is already normalized) +// It assumes len(sig) == 2*nLen where nLen is the byte-length of the curve order. +// This is needed when the underlying signature verification requires S to be in the lower range (to avoid signature malleability). In this package, verification allows high S signatures to be accepted. +// The function checks that S is in the correct range [0, n-1] before normalizing it. If S is not in the correct range, the function returns a false boolean. (S will be checked against 0 in the verification function - check against N is inlcuded here) +// returns: +// - newSig, true if S is in the valid range and was normalized to low S +// - nil, false if S was not in the correct range +func (a *ecdsaContext) signatureNormalizeS(sig []byte) ([]byte, bool) { // read S nLen := bitsToBytes(a.curveN.BitLen()) - s := new(big.Int).SetBytes(sig[nLen:]) - if a.isLowS(s) { - return sig // no need to flip S + s := new(big.Int).SetBytes(sig[nLen:]) // S >= 0 + if a.isLowS(s) { // S <= (n-1)/2 + return sig, true // S is in the valid range and no need to flip it } - // compute N-S - sComplement := new(big.Int).Sub(a.curveN, s) + + if a.curveN.Cmp(s) <= 0 { // S >= n, invalid signature + return nil, false + } + + // In the remaining case, (n-1)/2 < S < n and it is safe to flip + // i.e n-s is guaranteed to be in the range [1, (n-1)/2] + sComplement := new(big.Int).Sub(a.curveN, s) // n-S // write it into a new signature newSig := make([]byte, len(sig)) copy(newSig, sig[:nLen]) // copy R sComplement.FillBytes(newSig[nLen:]) // write S complement - return newSig + return newSig, true } // Test function only to flip S in a signature. It is used for testing signature malleability diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index de8811c0..027bef74 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -205,7 +205,10 @@ func (pk *pubKeyECDSASecp256k1) Verify(sig Signature, msg []byte, hasher hash.Ha // normalize the signature to low S. // This is required because the secp256k1 package does not accept high S signatures while the package allows them. // Rejecting high S signatures would be a breaking change with prior versions. - newSig := secp256k1Instance.signatureNormalizeS(sig) + newSig, validS := secp256k1Instance.signatureNormalizeS(sig) + if !validS { + return false, nil // S value is invalid, return early + } // truncate the hash to the curve order size, as specified in FIPS 186-4 section 6.4 (nLenSecp256k1 here is a multiple of 8 bits). // Moreover, the secp256k1 package requires the message hash to equal nLenSecp256k1 diff --git a/ecdsa_test.go b/ecdsa_test.go index 5c26f49c..10162374 100644 --- a/ecdsa_test.go +++ b/ecdsa_test.go @@ -419,58 +419,79 @@ func TestECDSAHighAndLowS(t *testing.T) { ECDSASecp256k1: secp256k1Instance, } - for _, curve := range ecdsaCurves { - t.Run(curve.String(), func(t *testing.T) { - // generate a key and sign a random message - seed := make([]byte, KeyGenSeedMinLen) - _, err := crand.Read(seed) - require.NoError(t, err) - sk, err := GeneratePrivateKey(curve, seed) - require.NoError(t, err) - - msg := make([]byte, 10) - _, err = crand.Read(msg) - require.NoError(t, err) - - halg := hash.NewSHA3_256() - sig, err := sk.Sign(msg, halg) - require.NoError(t, err) - - // extract S and test the first case of S (can be low or high S) - _, s := readTwoBigInts(sig, ecdsaSigLen[curve]/2) - isLowS := ecdsaContexts[curve].isLowS(s) - - t.Run(fmt.Sprintf("low S equals %v", isLowS), func(t *testing.T) { - // the format check must accept both forms - wellFormed, err := SignatureFormatCheck(curve, sig) + t.Run("lowS and HighS pass", func(t *testing.T) { + for _, curve := range ecdsaCurves { + t.Run(curve.String(), func(t *testing.T) { + // generate a key and sign a random message + seed := make([]byte, KeyGenSeedMinLen) + _, err := crand.Read(seed) require.NoError(t, err) - assert.True(t, wellFormed) - - // verification must accept the first form (can be low or high S) - valid, err := sk.PublicKey().Verify(sig, msg, halg) + sk, err := GeneratePrivateKey(curve, seed) require.NoError(t, err) - assert.True(t, valid) - }) - - // flip S to N-S to check the other case (can be low or high S) - t.Run(fmt.Sprintf("low S equals %v", !isLowS), func(t *testing.T) { - newSig := ecdsaContexts[curve].signatureFlipS(sig) - - // sanity check - _, newS := readTwoBigInts(newSig, ecdsaSigLen[curve]/2) - newIsLowS := ecdsaContexts[curve].isLowS(newS) - require.Equal(t, !newIsLowS, isLowS, "S didn't flip") // this test is correct because S cannot equal N-S since N is odd - // the format check must accept both forms - wellFormed, err := SignatureFormatCheck(curve, newSig) + msg := make([]byte, 10) + _, err = crand.Read(msg) require.NoError(t, err) - assert.True(t, wellFormed) - // verification must accept the second form (can be low or high S) - valid, err := sk.PublicKey().Verify(newSig, msg, halg) + halg := hash.NewSHA3_256() + sig, err := sk.Sign(msg, halg) require.NoError(t, err) - assert.True(t, valid) + + // extract S and test the first case of S (can be low or high S) + _, s := readTwoBigInts(sig, ecdsaSigLen[curve]/2) + isLowS := ecdsaContexts[curve].isLowS(s) + + t.Run(fmt.Sprintf("low S equals %v", isLowS), func(t *testing.T) { + // the format check must accept both forms + wellFormed, err := SignatureFormatCheck(curve, sig) + require.NoError(t, err) + assert.True(t, wellFormed) + + // verification must accept the first form (can be low or high S) + valid, err := sk.PublicKey().Verify(sig, msg, halg) + require.NoError(t, err) + assert.True(t, valid) + }) + + // flip S to N-S to check the other case (can be low or high S) + t.Run(fmt.Sprintf("low S equals %v", !isLowS), func(t *testing.T) { + newSig := ecdsaContexts[curve].signatureFlipS(sig) + + // sanity check + _, newS := readTwoBigInts(newSig, ecdsaSigLen[curve]/2) + newIsLowS := ecdsaContexts[curve].isLowS(newS) + require.Equal(t, !newIsLowS, isLowS, "S didn't flip") // this test is correct because S cannot equal N-S since N is odd + + // the format check must accept both forms + wellFormed, err := SignatureFormatCheck(curve, newSig) + require.NoError(t, err) + assert.True(t, wellFormed) + + // verification must accept the second form (can be low or high S) + valid, err := sk.PublicKey().Verify(newSig, msg, halg) + require.NoError(t, err) + assert.True(t, valid) + }) }) - }) - } + } + }) + + // signatureNormalizeS must reject values S >= N + t.Run("check signatureNormalizeS", func(t *testing.T) { + for _, curve := range ecdsaCurves { + t.Run(curve.String(), func(t *testing.T) { + nLen := ecdsaSigLen[curve] / 2 + badSig := make([]byte, ecdsaSigLen[curve]) + // set all S bytes to 0xFF which makes S larger than N. + // R value does not matter in the function + for i := nLen; i < len(badSig); i++ { + badSig[i] = 0xFF + } + + newSig, validS := ecdsaContexts[curve].signatureNormalizeS(badSig) + assert.False(t, validS) + assert.Nil(t, newSig) + }) + } + }) } From ab90dbfecf36dee3ee8537e9bbcb69fb356379a3 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Wed, 5 Aug 2026 17:01:04 -0500 Subject: [PATCH 12/21] downgrade go-eth to match flow-go version --- go.mod | 4 ++-- go.sum | 15 +++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index e5240a48..fb560979 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/onflow/crypto go 1.26.0 require ( - github.com/ethereum/go-ethereum v1.17.5 + github.com/ethereum/go-ethereum v1.16.8 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.54.0 @@ -15,6 +15,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/sys v0.47.0 // indirect - gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 8ad6fa05..29594720 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,19 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/ethereum/go-ethereum v1.17.5 h1:o9BIXs2Q/3cPHVxw49n+Zjn2i6rB9TOXatev46duOC4= -github.com/ethereum/go-ethereum v1.17.5/go.mod h1:vz2YvG7RewA4sFHTgzLyW+WmFG1N4jfk/hgXQVhhn9c= +github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9igY7law= +github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -25,8 +28,8 @@ golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 638803944530756298ae1681641695492fec1c39 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Fri, 7 Aug 2026 15:44:57 -0500 Subject: [PATCH 13/21] fix pub key decoding after downgrading go-eth crypto version and improve tests --- README.md | 2 +- blst_src/README.md | 2 +- ecdsa.go | 10 +++++---- ecdsa_p256.go | 4 +++- ecdsa_secp256k1.go | 38 ++++++++++++++++++++----------- ecdsa_test.go | 48 ++++++++++++++++++++++++++-------------- internal/blst/non_cgo.go | 3 --- 7 files changed, 68 insertions(+), 39 deletions(-) delete mode 100644 internal/blst/non_cgo.go diff --git a/README.md b/README.md index 4b912d94..65cd03e9 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,8 @@ All signature schemes use the generic interfaces of `PrivateKey` and `PublicKey` * ECDSA * public keys are compressed or uncompressed. - * ephemeral key is derived from the private key, hash and the system entropy (based on https://golang.org/pkg/crypto/ecdsa/). * supports NIST P-256 (secp256r1) and secp256k1 curves. + * For NIST P-256, ephemeral key is derived from the private key, hash and the system entropy (based on https://golang.org/pkg/crypto/ecdsa/). For secp256k1, ephemeral key is deterministically formed following RFC 6979 (based on github.com/ethereum/go-ethereum/crypto/secp256k1) * BLS * supports [BLS12-381](https://electriccoin.co/blog/new-snark-curve/) curve. diff --git a/blst_src/README.md b/blst_src/README.md index 48abe90e..1f4085f1 100644 --- a/blst_src/README.md +++ b/blst_src/README.md @@ -21,7 +21,7 @@ The folder contains: To upgrade the BLST version: - [ ] audit all BLST updates, with focus on `/src`: https://github.com/supranational/blst/compare/v0.3.14... - [ ] delete all files in this folder `./blst_src/` but `blst_src.c` and `README.md`. -- [ ] delete all files in `./internal/blst/` but `non_cgo.go`. +- [ ] delete all files in `./internal/blst/`. - [ ] open BLST repository on the new version. - [ ] copy all `.c` and `.h` files from `/src/` into `./blst_src/`. - [ ] delete newly copied `./blst_src/server.c`. diff --git a/ecdsa.go b/ecdsa.go index e7cf8cb0..0d3af994 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -41,7 +41,7 @@ type ecdsaContext struct { curveP *big.Int // curve order curveN *big.Int - // curve order minus 1 divided by 2 (used for signature malleability annalysis) + // curve order minus 1 divided by 2 (used for signature malleability analysis) curveNdiv2 *big.Int } @@ -122,7 +122,7 @@ func (a *ecdsaContext) mapToPrivateKey(seed []byte) (PrivateKey, error) { // privateKey returns an ECDSA private key using the // input scalar. // -// Input scalar d is assumed to be satisfy 0 < d < n before calling this function. +// Input scalar d is assumed to satisfy 0 < d < n before calling this function. // // The function returns: // - (nil, invalidInputsError) if the curve is not supported @@ -217,7 +217,7 @@ func (a *ecdsaContext) decodePrivateKey(der []byte) (PrivateKey, error) { func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { switch a.algo { case ECDSAP256: - return publicKeyECDSAP256(a, input) + return publicKeyECDSAP256(input) case ECDSASecp256k1: return publicKeyECDSASecp256k1(a, input) default: @@ -330,7 +330,9 @@ func (a *ecdsaContext) isLowS(s *big.Int) bool { // (same slice is returned if S is already normalized) // It assumes len(sig) == 2*nLen where nLen is the byte-length of the curve order. // This is needed when the underlying signature verification requires S to be in the lower range (to avoid signature malleability). In this package, verification allows high S signatures to be accepted. -// The function checks that S is in the correct range [0, n-1] before normalizing it. If S is not in the correct range, the function returns a false boolean. (S will be checked against 0 in the verification function - check against N is inlcuded here) +// The function checks that S is in the correct range [0, n-1] before normalizing it. +// If S is not in the correct range, the function returns a false boolean. +// (S will be checked against 0 in the verification function - check against N is inlcuded here) // returns: // - newSig, true if S is in the valid range and was normalized to low S // - nil, false if S was not in the correct range diff --git a/ecdsa_p256.go b/ecdsa_p256.go index 87f7b41f..d2838985 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -82,6 +82,7 @@ type pubKeyECDSAP256 struct { var _ PublicKey = (*pubKeyECDSAP256)(nil) +// Input scalar d is assumed to satisfy 0 < d < n before calling this function. func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error) { internalSK, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), dBytes) if err != nil { @@ -128,12 +129,13 @@ func (sk *prKeyECDSAP256) String() string { } // returns a publicKeyECDSAP256 from (bytes(x) || bytes(y)) bytes -func publicKeyECDSAP256(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSAP256, error) { +func publicKeyECDSAP256(XYBytes []byte) (*pubKeyECDSAP256, error) { // deserialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3) // and includes on curve check. // The bytes serialization for non-infinity points is `0x04 || X || Y` and infinity point should be rejected anyway parsingBytes := append([]byte{ecEncodingUncompressed}, XYBytes...) + // ParseUncompressedPublicKey includes x

= 0 || y.Cmp(a.curveP) >= 0 { + return nil, invalidInputsErrorf("at least one coordinate is larger than the field prime for %s", a.algo) + } + + // `IsOnCurve` includes checks for x

= p + ECDSASecp256k1, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F0000000000000000000000000000000000000000000000000000000000000000", + onflowCryptoErr, + }, { + ECDSAP256, "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000", + goCryptoErr, + }, { + // y >= p + ECDSASecp256k1, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC30", + onflowCryptoErr, + }, { + ECDSAP256, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", + goCryptoErr, + }, } - invalidPK2s := map[SigningAlgorithm]string{ - ECDSASecp256k1: "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F0000000000000000000000000000000000000000000000000000000000000000", - ECDSAP256: "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000", + + for _, invalidPK := range invalidPKs { + pkBytes, err := hex.DecodeString(invalidPK.pk) + require.NoError(t, err) + pk, err := DecodePublicKey(invalidPK.signin, pkBytes) + require.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + assert.ErrorContains(t, err, invalidPK.errorMsg) + assert.Nil(t, pk) } - // invalidpk1 with x >= p - invalidPk1, err := hex.DecodeString(invalidPK1s[curve]) - require.NoError(t, err) - _, err = DecodePublicKey(curve, invalidPk1) - assert.Error(t, err) - // invalidpk2 with y >= p - invalidPk2, err := hex.DecodeString(invalidPK2s[curve]) - require.NoError(t, err) - _, err = DecodePublicKey(curve, invalidPk2) - assert.Error(t, err) }) } } diff --git a/internal/blst/non_cgo.go b/internal/blst/non_cgo.go deleted file mode 100644 index 324387c6..00000000 --- a/internal/blst/non_cgo.go +++ /dev/null @@ -1,3 +0,0 @@ -//go:build !cgo && no_cgo - -package blst From 2ff230529066c0ef030693344fe91afef7beb92b Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 10 Aug 2026 13:23:34 -0500 Subject: [PATCH 14/21] minor improvements --- ecdsa.go | 18 ++++--- ecdsa_p256.go | 4 +- ecdsa_secp256k1.go | 10 ++-- ecdsa_test.go | 123 +++++++++++++++++++++++++-------------------- go.mod | 6 ++- go.sum | 32 ++++++++++++ 6 files changed, 123 insertions(+), 70 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index 0d3af994..841486c1 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -76,8 +76,12 @@ func (a *ecdsaContext) checkAlgoAndComputeHash(msg []byte, hasher hash.Hasher) ( // signatureFormatCheck verifies the format of a serialized signature, // regardless of messages or public keys. -// If FormatCheck returns false then the input is not a valid ECDSA +// If signatureFormatCheck returns false then the input is not a valid ECDSA // signature and will fail a verification against any message and public key. +// +// This function is not called for signature verification. Checks of signature +// components R and S are delegated to the verification functions of the underlying +// packages. func (a *ecdsaContext) signatureFormatCheck(sig Signature) bool { N := a.curveN nLen := bitsToBytes(N.BitLen()) @@ -274,7 +278,7 @@ func pubKeyCommonECDSAString(pk PublicKey) string { return fmt.Sprintf("%#x", pk.Encode()) } -// Equals test the equality of two private keys +// Equals tests the equality of two private keys func prKeyCommonECDSAEquals(sk, other PrivateKey) bool { // check the algorithm if sk.Algorithm() != other.Algorithm() { @@ -294,7 +298,7 @@ func (pk *pubKeyCommonECDSA) Size() int { return 2 * bitsToBytes(pk.curveP.BitLen()) } -// Equals test the equality of two private keys +// Equals tests the equality of two private keys func pubKeyCommonECDSAEquals(pk, other PublicKey) bool { // check the algorithm if pk.Algorithm() != other.Algorithm() { @@ -312,7 +316,7 @@ func padToSizeAndConcat(output []byte, a, b *big.Int, size int) { b.FillBytes(output[size:]) } -// Helper function to read two big integers of "size" bytes each from a concatenate input buffer. +// Helper function to read two big integers of "size" bytes each from a concatenated input buffer. // This helper is needed when deserializing. // It assumes the input buffer has at least 2*size byte-length. func readTwoBigInts(input []byte, size int) (*big.Int, *big.Int) { @@ -330,9 +334,9 @@ func (a *ecdsaContext) isLowS(s *big.Int) bool { // (same slice is returned if S is already normalized) // It assumes len(sig) == 2*nLen where nLen is the byte-length of the curve order. // This is needed when the underlying signature verification requires S to be in the lower range (to avoid signature malleability). In this package, verification allows high S signatures to be accepted. -// The function checks that S is in the correct range [0, n-1] before normalizing it. -// If S is not in the correct range, the function returns a false boolean. -// (S will be checked against 0 in the verification function - check against N is inlcuded here) +// The function checks that S is in the range [0, n-1] before normalizing it. +// If S is not in this range, the function returns a false boolean. +// (S and R values will be checked by the go-ethereum verification function - only S check against N is included here, S=0 check is deferred to the signature verification) // returns: // - newSig, true if S is in the valid range and was normalized to low S // - nil, false if S was not in the correct range diff --git a/ecdsa_p256.go b/ecdsa_p256.go index d2838985..5a890c3e 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -206,12 +206,12 @@ func (sk *prKeyECDSAP256) Encode() []byte { return sk.rawEncode() } -// Equals test the equality of two private keys +// Equals tests the equality of two private keys func (sk *prKeyECDSAP256) Equals(other PrivateKey) bool { return prKeyCommonECDSAEquals(sk, other) } -// Equals test the equality of two public keys +// Equals tests the equality of two public keys func (pk *pubKeyECDSAP256) Equals(other PublicKey) bool { return pubKeyCommonECDSAEquals(pk, other) } diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index 54821f52..1de5d22a 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -97,7 +97,7 @@ type pubKeyECDSASecp256k1 struct { var _ PublicKey = (*pubKeyECDSASecp256k1)(nil) -// Input scalar d is assumed to be satisfy 0 < d < n before calling this function. +// Input scalar d is assumed to satisfy 0 < d < n before calling this function. func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256k1 { sk := &prKeyECDSASecp256k1{ prKeyCommonECDSA: &prKeyCommonECDSA{a}, @@ -150,12 +150,12 @@ func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp2 x, y := readTwoBigInts(XYBytes, pLen) - // check the coordinates are valid field elements (required for go-ethereum versions prior or equal to v1.16.8) + // check the coordinates are valid field elements (required for go-ethereum versions prior to or equal to v1.16.8) if x.Cmp(a.curveP) >= 0 || y.Cmp(a.curveP) >= 0 { return nil, invalidInputsErrorf("at least one coordinate is larger than the field prime for %s", a.algo) } - // `IsOnCurve` includes checks for x

= p - ECDSASecp256k1, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F0000000000000000000000000000000000000000000000000000000000000000", - onflowCryptoErr, - }, { - ECDSAP256, "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000", - goCryptoErr, - }, { - // y >= p - ECDSASecp256k1, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC30", - onflowCryptoErr, - }, { - ECDSAP256, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", - goCryptoErr, - }, - } - - for _, invalidPK := range invalidPKs { - pkBytes, err := hex.DecodeString(invalidPK.pk) - require.NoError(t, err) - pk, err := DecodePublicKey(invalidPK.signin, pkBytes) - require.Error(t, err) - assert.True(t, IsInvalidInputsError(err)) - assert.ErrorContains(t, err, invalidPK.errorMsg) - assert.Nil(t, pk) - } - }) } + // Test a public key serialization with a point encoded with + // x or y not reduced mod p. + // This test checks that: + // - public key decoding handles input x-coordinates with x and y larger than p (doesn't result in an exception) + // - public key decoding only accepts reduced x and y + t.Run("public key with non-reduced coordinates", func(t *testing.T) { + onflowCryptoErr := "at least one coordinate is larger than the field prime" + goCryptoErr := "invalid P256 element encoding" + + invalidPKs := []struct { + curve SigningAlgorithm + pk string + errorMsg string + // assertions are based on the correct error message. + // In particular, the error message in this test must be about the coordinates + // being incorrect/non-reduced rather than the point not being on curve. + // Future edits must not update the error messages without taking this into account. + }{ + // x >= p , point not on curve + { + ECDSASecp256k1, "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F0000000000000000000000000000000000000000000000000000000000000000", + onflowCryptoErr, + }, { + ECDSAP256, "FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF0000000000000000000000000000000000000000000000000000000000000000", + goCryptoErr, + }, + // y >= p , point not on curve + { + ECDSASecp256k1, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC30", + onflowCryptoErr, + }, { + ECDSAP256, "0000000000000000000000000000000000000000000000000000000000000000FFFFFFFF00000001000000000000000000000000FFFFFFFFFFFFFFFFFFFFFFFF", + goCryptoErr, + }, + // x >= p , point on curve + { + ECDSASecp256k1, "fffffffffffffffffffffffffffffffffffffffffffffffffffffffefffffc304218f20ae6c646b363db68605822fb14264ca8d2587fdd6fbc750d587e76a7ee", + onflowCryptoErr, + }, + } + + for _, invalidPK := range invalidPKs { + pkBytes, err := hex.DecodeString(invalidPK.pk) + require.NoError(t, err) + pk, err := DecodePublicKey(invalidPK.curve, pkBytes) + require.Error(t, err) + assert.True(t, IsInvalidInputsError(err)) + assert.ErrorContains(t, err, invalidPK.errorMsg) + assert.Nil(t, pk) + } + }) } // TestECDSAEquals tests equal for ECDSA keys @@ -281,10 +289,10 @@ func TestECDSAPublicKeyComputation(t *testing.T) { } } +// TestECDSASignatureFormatCheck tests SignatureFormatCheck. func TestECDSASignatureFormatCheck(t *testing.T) { - for _, curve := range ecdsaCurves { - t.Run("valid signature", func(t *testing.T) { + t.Run("valid signature check", func(t *testing.T) { len := ecdsaSigLen[curve] sig := Signature(make([]byte, len)) _, err := crand.Read(sig) @@ -310,9 +318,8 @@ func TestECDSASignatureFormatCheck(t *testing.T) { assert.Nil(t, err) assert.False(t, valid) }) - t.Run("zero values", func(t *testing.T) { - // signature with a zero s + // S=0 len := ecdsaSigLen[curve] sig0s := Signature(make([]byte, len)) _, err := crand.Read(sig0s[:len/2]) @@ -322,7 +329,7 @@ func TestECDSASignatureFormatCheck(t *testing.T) { assert.Nil(t, err) assert.False(t, valid) - // signature with a zero r + // R=0 sig0r := Signature(make([]byte, len)) _, err = crand.Read(sig0r[len/2:]) require.NoError(t, err) @@ -330,14 +337,20 @@ func TestECDSASignatureFormatCheck(t *testing.T) { valid, err = SignatureFormatCheck(curve, sig0r) assert.Nil(t, err) assert.False(t, valid) + + // signature with R=S=0 + sig0 := Signature(make([]byte, len)) + valid, err = SignatureFormatCheck(curve, sig0) + assert.Nil(t, err) + assert.False(t, valid) }) - t.Run("large values", func(t *testing.T) { + t.Run("non-reduced values", func(t *testing.T) { len := ecdsaSigLen[curve] sigLargeS := Signature(make([]byte, len)) _, err := crand.Read(sigLargeS[:len/2]) require.NoError(t, err) - // make sure s is larger than the curve order + // s >= N for i := len / 2; i < len; i++ { sigLargeS[i] = 0xFF } @@ -349,7 +362,7 @@ func TestECDSASignatureFormatCheck(t *testing.T) { sigLargeR := Signature(make([]byte, len)) _, err = crand.Read(sigLargeR[len/2:]) require.NoError(t, err) - // make sure s is larger than the curve order + // R >= N for i := 0; i < len/2; i++ { sigLargeR[i] = 0xFF } @@ -427,7 +440,7 @@ func TestECDSAKeyGenerationBreakingChange(t *testing.T) { // // For a valid signature (r,s), the pair (r,n-s) is also a valid signature of the same // message under the same key. The package signature verification accepts both forms and should keep doing so. -// Rejecting the high-s form would be a breaking change for the applications using this package. +// Rejecting the high-s form would be a breaking change for applications using this package. func TestECDSAHighAndLowS(t *testing.T) { var ecdsaContexts = map[SigningAlgorithm]*ecdsaContext{ @@ -453,7 +466,7 @@ func TestECDSAHighAndLowS(t *testing.T) { sig, err := sk.Sign(msg, halg) require.NoError(t, err) - // extract S and test the first case of S (can be low or high S) + // extract S and test the current case of S (either low or high) _, s := readTwoBigInts(sig, ecdsaSigLen[curve]/2) isLowS := ecdsaContexts[curve].isLowS(s) @@ -469,7 +482,7 @@ func TestECDSAHighAndLowS(t *testing.T) { assert.True(t, valid) }) - // flip S to N-S to check the other case (can be low or high S) + // flip S to N-S to check the other case t.Run(fmt.Sprintf("low S equals %v", !isLowS), func(t *testing.T) { newSig := ecdsaContexts[curve].signatureFlipS(sig) diff --git a/go.mod b/go.mod index fb560979..c45aa35a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,11 @@ module github.com/onflow/crypto go 1.26.0 require ( - github.com/ethereum/go-ethereum v1.16.8 + github.com/ethereum/go-ethereum v1.16.8 + // fixed at this version because flow-go uses this version and Go modules does not allow + // multiple versions of the same module. + // This will be updated once flow-go updates to a newer version of go-ethereum. + // Updates must audit the code changes under github.com/ethereum/go-ethereum/crypto/secp256k1 between the old and new version. github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.54.0 diff --git a/go.sum b/go.sum index 29594720..768e6860 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,26 @@ +github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= +github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= +github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= +github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/ethereum/c-kzg-4844/v2 v2.1.8 h1:oQ48q/TMe2SKU8qBE3N7e4/HlG3EpJftom6EsPQgJ58= +github.com/ethereum/c-kzg-4844/v2 v2.1.8/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9igY7law= github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= +github.com/ethereum/go-ethereum v1.17.5 h1:o9BIXs2Q/3cPHVxw49n+Zjn2i6rB9TOXatev46duOC4= +github.com/ethereum/go-ethereum v1.17.5/go.mod h1:vz2YvG7RewA4sFHTgzLyW+WmFG1N4jfk/hgXQVhhn9c= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -10,23 +28,35 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= +github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -35,3 +65,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= +pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= +pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= From 7de9949b39cb3bca54f96eff029f2295ab270551 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 10 Aug 2026 13:26:41 -0500 Subject: [PATCH 15/21] fix doc errors --- ecdsa_p256.go | 4 ++-- ecdsa_secp256k1.go | 4 ++-- spock.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ecdsa_p256.go b/ecdsa_p256.go index 5a890c3e..a5b8e02f 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -104,8 +104,8 @@ func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error // modified temporarily. // // The function returns: -// - (false, errNilHasher) if a hasher is nil -// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (nil, errNilHasher) if a hasher is nil +// - (nil, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). // - (nil, error) if an unexpected error occurs // - (signature, nil) otherwise func (sk *prKeyECDSAP256) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index 1de5d22a..f5293a65 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -115,8 +115,8 @@ func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256 // modified temporarily. // // The function returns: -// - (false, errNilHasher) if a hasher is nil -// - (false, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). +// - (nil, errNilHasher) if a hasher is nil +// - (nil, invalidHasherSizeError) when the hasher's output size is less than the curve order (currently 32 bytes). // - (nil, error) if an unexpected error occurs // - (signature, nil) otherwise func (sk *prKeyECDSASecp256k1) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { diff --git a/spock.go b/spock.go index 5927c9be..c82e21a9 100644 --- a/spock.go +++ b/spock.go @@ -32,8 +32,8 @@ import ( // SPOCKProve generates a spock poof for data under the private key sk. // // The function returns: -// - (false, errNilHasher) if the hasher is nil -// - (false, invalidHasherSiseError) if hasher's output size is not 128 bytes +// - (nil, errNilHasher) if the hasher is nil +// - (nil, invalidHasherSiseError) if hasher's output size is not 128 bytes // - (nil, errNotBLSKey) if input key is not a BLS key // - (nil, error) if an unexpected error occurs // - (proof, nil) otherwise From 16739683111115ea7824aa5e4e9de3d5acc519f7 Mon Sep 17 00:00:00 2001 From: Tarak Ben Youssef Date: Mon, 10 Aug 2026 13:48:44 -0500 Subject: [PATCH 16/21] mod tidy --- go.mod | 2 +- go.sum | 32 -------------------------------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/go.mod b/go.mod index c45aa35a..785459c4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/onflow/crypto go 1.26.0 require ( - github.com/ethereum/go-ethereum v1.16.8 + github.com/ethereum/go-ethereum v1.16.8 // fixed at this version because flow-go uses this version and Go modules does not allow // multiple versions of the same module. // This will be updated once flow-go updates to a newer version of go-ethereum. diff --git a/go.sum b/go.sum index 768e6860..29594720 100644 --- a/go.sum +++ b/go.sum @@ -1,26 +1,8 @@ -github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= -github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= -github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= -github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= -github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= -github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/ethereum/c-kzg-4844/v2 v2.1.8 h1:oQ48q/TMe2SKU8qBE3N7e4/HlG3EpJftom6EsPQgJ58= -github.com/ethereum/c-kzg-4844/v2 v2.1.8/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-ethereum v1.16.8 h1:LLLfkZWijhR5m6yrAXbdlTeXoqontH+Ga2f9igY7law= github.com/ethereum/go-ethereum v1.16.8/go.mod h1:Fs6QebQbavneQTYcA39PEKv2+zIjX7rPUZ14DER46wk= -github.com/ethereum/go-ethereum v1.17.5 h1:o9BIXs2Q/3cPHVxw49n+Zjn2i6rB9TOXatev46duOC4= -github.com/ethereum/go-ethereum v1.17.5/go.mod h1:vz2YvG7RewA4sFHTgzLyW+WmFG1N4jfk/hgXQVhhn9c= -github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= -github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -28,35 +10,23 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= -github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -65,5 +35,3 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= pgregory.net/rapid v0.4.7 h1:MTNRktPuv5FNqOO151TM9mDTa+XHcX6ypYeISDVD14g= pgregory.net/rapid v0.4.7/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU= -pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= -pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= From 5abede86f980891b1ae371138efb9fb21f8321b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 10 Aug 2026 13:27:43 -0700 Subject: [PATCH 17/21] improve comments --- ecdsa_secp256k1.go | 7 ++++++- go.mod | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index f5293a65..11453a38 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -150,7 +150,12 @@ func publicKeyECDSASecp256k1(a *ecdsaContext, XYBytes []byte) (*pubKeyECDSASecp2 x, y := readTwoBigInts(XYBytes, pLen) - // check the coordinates are valid field elements (required for go-ethereum versions prior to or equal to v1.16.8) + // check the coordinates are valid field elements. + // This check is unconditional and must not be removed when go-ethereum is upgraded: + // it keeps the set of accepted key encodings independent of the go-ethereum version + // that Go module resolution selects, which this package does not control. + // Without it, a non-canonical encoding such as `x+p` is accepted whenever + // the resolved go-ethereum is v1.16.8 or earlier. if x.Cmp(a.curveP) >= 0 || y.Cmp(a.curveP) >= 0 { return nil, invalidInputsErrorf("at least one coordinate is larger than the field prime for %s", a.algo) } diff --git a/go.mod b/go.mod index 785459c4..88ee2b44 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,12 @@ module github.com/onflow/crypto go 1.26.0 require ( + // Minimum version, not a pin: Go builds with the highest version required across the build. + // Keeping this floor low lets consumers choose their own go-ethereum version; + // raising it forces the new version on all of them. + // Only raise it if this module needs something v1.16.8 lacks, + // and audit the changes under go-ethereum/crypto/secp256k1 when doing so. github.com/ethereum/go-ethereum v1.16.8 - // fixed at this version because flow-go uses this version and Go modules does not allow - // multiple versions of the same module. - // This will be updated once flow-go updates to a newer version of go-ethereum. - // Updates must audit the code changes under github.com/ethereum/go-ethereum/crypto/secp256k1 between the old and new version. github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.54.0 From 50ddda0116bb6f38f3270ab8529c1e6a6d4b8c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Mon, 10 Aug 2026 14:36:13 -0700 Subject: [PATCH 18/21] fix ECDSA edge cases and harden new implementation - return untyped nil interfaces on key decoding errors - return false instead of panicking in Equals(nil) - reject hashers whose computed hash is shorter than their declared size - clip the capacity of secp256k1 signatures sliced from the [R||S||V] buffer - make lazy PublicKey() construction safe for concurrent use - derive (N-1)/2 from N and cross-check the secp256k1 constants against go-ethereum at initialization - restore the P-256 public key length check - add tests: RFC 6979 known-answer vectors, concurrent PublicKey, nil inputs, dishonest hasher, compressed edge points --- ecdsa.go | 92 +++++++++++++++++------- ecdsa_p256.go | 38 +++++----- ecdsa_secp256k1.go | 38 ++++++---- ecdsa_test.go | 171 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 282 insertions(+), 57 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index 841486c1..ffbcfffd 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -58,19 +58,29 @@ func bitsToBytes(bits int) int { return (bits + 7) >> 3 } -func (a *ecdsaContext) checkAlgoAndComputeHash(msg []byte, hasher hash.Hasher) (hash.Hash, error) { +// checkHasherAndComputeHash checks the hasher is valid for ECDSA +// on the receiver curve and returns the hash of the input message. +func (a *ecdsaContext) checkHasherAndComputeHash(msg []byte, hasher hash.Hasher) (hash.Hash, error) { if hasher == nil { return nil, errNilHasher } // check hasher's size is at least the curve order in bytes - nLen := (a.curveN).BitLen() - if (hasher.Size() << 3) < nLen { + nLen := bitsToBytes((a.curveN).BitLen()) + if hasher.Size() < nLen { return nil, invalidHasherSizeErrorf( - "hasher's bit-size should be at least %d, got %d", nLen, hasher.Size()<<3) + "hasher's size should be at least %d bytes, got %d bytes", nLen, hasher.Size()) } h := hasher.ComputeHash(msg) + // guard against hasher implementations that compute fewer bytes + // than their declared size, + // since callers truncate the hash to the curve order size + // and would panic on a shorter slice + if len(h) < nLen { + return nil, invalidHasherSizeErrorf( + "hasher's output should be at least %d bytes, got %d bytes", nLen, len(h)) + } return h, nil } @@ -106,7 +116,6 @@ func (a *ecdsaContext) signatureFormatCheck(sig Signature) bool { } var one = new(big.Int).SetInt64(1) -var two = new(big.Int).SetInt64(2) // mapToPrivateKey simply maps the input seed to an ECDSA private key // The private scalar `d` satisfies 0 < d < n. @@ -139,7 +148,14 @@ func (a *ecdsaContext) privateKey(d *big.Int) (PrivateKey, error) { // build the private key depending on the curve switch a.algo { case ECDSAP256: - return privateKeyECDSAP256(a, dBytes) + sk, err := privateKeyECDSAP256(a, dBytes) + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return sk, nil case ECDSASecp256k1: return privateKeyECDSASecp256k1(a, dBytes), nil default: @@ -221,9 +237,23 @@ func (a *ecdsaContext) decodePrivateKey(der []byte) (PrivateKey, error) { func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { switch a.algo { case ECDSAP256: - return publicKeyECDSAP256(input) + pk, err := publicKeyECDSAP256(input) + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil case ECDSASecp256k1: - return publicKeyECDSASecp256k1(a, input) + pk, err := publicKeyECDSASecp256k1(a, input) + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil default: return nil, invalidInputsErrorf("curve is not supported") } @@ -243,9 +273,23 @@ func (a *ecdsaContext) decodePublicKey(der []byte) (PublicKey, error) { func (a *ecdsaContext) decodePublicKeyCompressed(pkBytes []byte) (PublicKey, error) { switch a.algo { case ECDSAP256: - return p256DecodePublicKeyCompressed(pkBytes) + pk, err := p256DecodePublicKeyCompressed(pkBytes) + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil case ECDSASecp256k1: - return secp256k1DecodePublicKeyCompressed(pkBytes) + pk, err := secp256k1DecodePublicKeyCompressed(pkBytes) + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil default: return nil, invalidInputsErrorf("the input curve is not supported") } @@ -280,6 +324,10 @@ func pubKeyCommonECDSAString(pk PublicKey) string { // Equals tests the equality of two private keys func prKeyCommonECDSAEquals(sk, other PrivateKey) bool { + // a nil key is not equal to any key + if other == nil { + return false + } // check the algorithm if sk.Algorithm() != other.Algorithm() { return false @@ -300,6 +348,10 @@ func (pk *pubKeyCommonECDSA) Size() int { // Equals tests the equality of two private keys func pubKeyCommonECDSAEquals(pk, other PublicKey) bool { + // a nil key is not equal to any key + if other == nil { + return false + } // check the algorithm if pk.Algorithm() != other.Algorithm() { return false @@ -313,7 +365,7 @@ func pubKeyCommonECDSAEquals(pk, other PublicKey) bool { // It assumes the output buffer has at least 2*size byte-length func padToSizeAndConcat(output []byte, a, b *big.Int, size int) { a.FillBytes(output[:size]) - b.FillBytes(output[size:]) + b.FillBytes(output[size : 2*size]) } // Helper function to read two big integers of "size" bytes each from a concatenated input buffer. @@ -333,7 +385,9 @@ func (a *ecdsaContext) isLowS(s *big.Int) bool { // signatureNormalizeS returns a signature with S normalized to low S. // (same slice is returned if S is already normalized) // It assumes len(sig) == 2*nLen where nLen is the byte-length of the curve order. -// This is needed when the underlying signature verification requires S to be in the lower range (to avoid signature malleability). In this package, verification allows high S signatures to be accepted. +// This is needed when the underlying signature verification requires S to be +// in the lower range (to avoid signature malleability). +// In this package, verification allows high S signatures to be accepted. // The function checks that S is in the range [0, n-1] before normalizing it. // If S is not in this range, the function returns a false boolean. // (S and R values will be checked by the go-ethereum verification function - only S check against N is included here, S=0 check is deferred to the signature verification) @@ -361,17 +415,3 @@ func (a *ecdsaContext) signatureNormalizeS(sig []byte) ([]byte, bool) { sComplement.FillBytes(newSig[nLen:]) // write S complement return newSig, true } - -// Test function only to flip S in a signature. It is used for testing signature malleability -func (a *ecdsaContext) signatureFlipS(sig []byte) []byte { - // read S - nLen := bitsToBytes(a.curveN.BitLen()) - s := new(big.Int).SetBytes(sig[nLen:]) - // compute N-S - sComplement := new(big.Int).Sub(a.curveN, s) - // write it into a new signature - newSig := make([]byte, len(sig)) - copy(newSig, sig[:nLen]) // copy R - sComplement.FillBytes(newSig[nLen:]) // write S complement - return newSig -} diff --git a/ecdsa_p256.go b/ecdsa_p256.go index a5b8e02f..2f218277 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -24,6 +24,7 @@ import ( "crypto/rand" "fmt" "math/big" + "sync" "github.com/onflow/crypto/hash" ) @@ -50,12 +51,11 @@ var p256Instance *ecdsaContext func initECDSAP256() { curve := elliptic.P256() n := curve.Params().N - nMinus1 := new(big.Int).Sub(n, one) p256Instance = &(ecdsaContext{ curveP: curve.Params().P, curveN: n, - curveNdiv2: new(big.Int).Div(nMinus1, two), // (N-1)/2 + curveNdiv2: new(big.Int).Rsh(n, 1), // (N-1)/2, since N is odd algo: ECDSAP256, }) } @@ -66,6 +66,9 @@ type prKeyECDSAP256 struct { *prKeyCommonECDSA // go ecdsa standard lib private key goPrKey *ecdsa.PrivateKey + // pubKeyOnce guards the lazy construction of pubKey, + // making concurrent calls to PublicKey safe + pubKeyOnce sync.Once // public key pubKey *pubKeyECDSAP256 } @@ -109,7 +112,7 @@ func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error // - (nil, error) if an unexpected error occurs // - (signature, nil) otherwise func (sk *prKeyECDSAP256) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { - hash, err := sk.checkAlgoAndComputeHash(msg, hasher) + hash, err := sk.checkHasherAndComputeHash(msg, hasher) if err != nil { return nil, err } @@ -130,6 +133,11 @@ func (sk *prKeyECDSAP256) String() string { // returns a publicKeyECDSAP256 from (bytes(x) || bytes(y)) bytes func publicKeyECDSAP256(XYBytes []byte) (*pubKeyECDSAP256, error) { + if len(XYBytes) != 2*pLenP256 { + return nil, invalidInputsErrorf("input has incorrect %s key size, got %d, expects %d", + ECDSAP256, len(XYBytes), 2*pLenP256) + } + // deserialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3) // and includes on curve check. // The bytes serialization for non-infinity points is `0x04 || X || Y` and infinity point should be rejected anyway @@ -154,12 +162,12 @@ func (pk *pubKeyECDSAP256) String() string { // PublicKey returns the public key associated to the private key func (sk *prKeyECDSAP256) PublicKey() PublicKey { // construct the public key once - if sk.pubKey == nil { + sk.pubKeyOnce.Do(func() { sk.pubKey = &pubKeyECDSAP256{ pubKeyCommonECDSA: &pubKeyCommonECDSA{p256Instance}, goPubKey: &sk.goPrKey.PublicKey, } - } + }) return sk.pubKey } @@ -177,7 +185,7 @@ func (sk *prKeyECDSAP256) PublicKey() PublicKey { // - (false, error) if an unexpected error occurs // - (validity, nil) otherwise func (pk *pubKeyECDSAP256) Verify(sig Signature, data []byte, alg hash.Hasher) (bool, error) { - h, err := pk.checkAlgoAndComputeHash(data, alg) + h, err := pk.checkHasherAndComputeHash(data, alg) if err != nil { return false, err } @@ -266,17 +274,9 @@ func p256DecodePublicKeyCompressed(pkBytes []byte) (*pubKeyECDSAP256, error) { if x == nil || y == nil { return nil, invalidInputsErrorf("input %x isn't a compressed serialization of a point on P256", pkBytes) } - uncompressedPointBytes := make([]byte, 2*pLenP256+1) - uncompressedPointBytes[0] = ecEncodingUncompressed - padToSizeAndConcat(uncompressedPointBytes[1:], x, y, pLenP256) - - internalPK, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), uncompressedPointBytes) - if err != nil { - // unexpected error since prior deserialization succeeded - return nil, invalidInputsErrorf("unexpected error: input is not a point on curve P-256: %w", err) - } - return &pubKeyECDSAP256{ - &pubKeyCommonECDSA{p256Instance}, - internalPK, - }, nil + // serialize the coordinates and delegate to the uncompressed decoding, + // so that both decoding paths construct the key the same way + xyBytes := make([]byte, 2*pLenP256) + padToSizeAndConcat(xyBytes, x, y, pLenP256) + return publicKeyECDSAP256(xyBytes) } diff --git a/ecdsa_secp256k1.go b/ecdsa_secp256k1.go index 11453a38..8c3c1eae 100644 --- a/ecdsa_secp256k1.go +++ b/ecdsa_secp256k1.go @@ -21,6 +21,7 @@ package crypto import ( "fmt" "math/big" + "sync" "github.com/ethereum/go-ethereum/crypto/secp256k1" @@ -38,8 +39,6 @@ const ( nLenSecp256k1 = 32 pLenSecp256k1 = 32 - - secp256k1Ndiv2Hex = "7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF5D576E7357A4501DDFE92F46681B20A0" ) const ( @@ -62,14 +61,16 @@ func initECDSASecp256k1() { if !ok { panic("failed to initialize ECDSA with secp256k1 curve") } - curveNdiv2, ok := new(big.Int).SetString(secp256k1Ndiv2Hex, 16) - if !ok { - panic("failed to initialize ECDSA with secp256k1 curve") + // cross-check the hard-coded SEC2 constants against the constants + // of the underlying go-ethereum implementation, + // so that a typo in either place is caught at initialization + if curveP.Cmp(secp256k1.S256().P) != 0 || curveN.Cmp(secp256k1.S256().N) != 0 { + panic("secp256k1 curve constants do not match the underlying go-ethereum implementation") } secp256k1Instance = &(ecdsaContext{ curveP: curveP, curveN: curveN, - curveNdiv2: curveNdiv2, + curveNdiv2: new(big.Int).Rsh(curveN, 1), // (N-1)/2, since N is odd algo: ECDSASecp256k1, }) } @@ -80,6 +81,9 @@ type prKeyECDSASecp256k1 struct { *prKeyCommonECDSA // bytes(D) of private scalar D in big endian, padded to the curve order size (32 bytes) dBytes []byte + // pubKeyOnce guards the lazy construction of pubKey, + // making concurrent calls to PublicKey safe + pubKeyOnce sync.Once // public key pubKey *pubKeyECDSASecp256k1 } @@ -120,7 +124,7 @@ func privateKeyECDSASecp256k1(a *ecdsaContext, dBytes []byte) *prKeyECDSASecp256 // - (nil, error) if an unexpected error occurs // - (signature, nil) otherwise func (sk *prKeyECDSASecp256k1) Sign(msg []byte, hasher hash.Hasher) (Signature, error) { - hash, err := sk.checkAlgoAndComputeHash(msg, hasher) + hash, err := sk.checkHasherAndComputeHash(msg, hasher) if err != nil { return nil, err } @@ -131,8 +135,10 @@ func (sk *prKeyECDSASecp256k1) Sign(msg []byte, hasher hash.Hasher) (Signature, if err != nil { return nil, fmt.Errorf("failed to sign hash: %w", err) } - // remove the EC recover byte (last byte) - return signature[:SignatureLenECDSASecp256k1], nil + // remove the EC recover byte (last byte). + // The capacity is clipped so that an append to the returned signature + // cannot reach the recovery byte in the shared backing array. + return signature[:SignatureLenECDSASecp256k1:SignatureLenECDSASecp256k1], nil } // String returns the hex string representation of the private key @@ -190,14 +196,22 @@ func secp256k1PkBytes(x, y *big.Int) []byte { // PublicKey returns the public key associated to the private key func (sk *prKeyECDSASecp256k1) PublicKey() PublicKey { // construct the public key once - if sk.pubKey == nil { + sk.pubKeyOnce.Do(func() { x, y := secp256k1.S256().ScalarBaseMult(sk.dBytes) + // `ScalarBaseMult` returns nil coordinates only if the scalar is zero + // or not less than the curve order, + // which all constructors of `prKeyECDSASecp256k1` rule out. + // The check turns an unexpected invariant break into an explicit panic + // instead of a nil dereference inside `secp256k1PkBytes`. + if x == nil || y == nil { + panic("unexpected error: the private key scalar is invalid") + } sk.pubKey = &pubKeyECDSASecp256k1{ pubKeyCommonECDSA: &pubKeyCommonECDSA{secp256k1Instance}, pkBytes: secp256k1PkBytes(x, y), } - } + }) return sk.pubKey } @@ -215,7 +229,7 @@ func (sk *prKeyECDSASecp256k1) PublicKey() PublicKey { // - (false, error) if an unexpected error occurs // - (validity, nil) otherwise func (pk *pubKeyECDSASecp256k1) Verify(sig Signature, msg []byte, hasher hash.Hasher) (bool, error) { - hash, err := pk.checkAlgoAndComputeHash(msg, hasher) + hash, err := pk.checkHasherAndComputeHash(msg, hasher) if err != nil { return false, err } diff --git a/ecdsa_test.go b/ecdsa_test.go index 26f5f857..8c9e3c35 100644 --- a/ecdsa_test.go +++ b/ecdsa_test.go @@ -21,6 +21,8 @@ package crypto import ( "encoding/hex" "fmt" + "math/big" + "sync" "testing" crand "crypto/rand" @@ -70,6 +72,13 @@ func (d *dummyHasher) Write([]byte) (int, error) { return 0, nil } func (d *dummyHasher) SumHash() hash.Hash { return make([]byte, d.size) } func (d *dummyHasher) Reset() {} +// dishonestHasher declares a size but computes hashes one byte shorter, +// simulating a hash.Hasher implementation that breaks the interface contract +type dishonestHasher struct{ dummyHasher } + +func newDishonestHasher(size int) hash.Hasher { return &dishonestHasher{dummyHasher{size}} } +func (d *dishonestHasher) ComputeHash([]byte) hash.Hash { return make([]byte, d.size-1) } + func TestECDSAHasher(t *testing.T) { for _, curve := range ecdsaCurves { // generate a key pair @@ -110,6 +119,17 @@ func TestECDSAHasher(t *testing.T) { assert.Error(t, err) assert.True(t, IsInvalidHasherSizeError(err)) }) + + // hasher whose computed hash is shorter than its declared size + t.Run("dishonest hasher is rejected without a panic", func(t *testing.T) { + dummy := newDishonestHasher(32) + _, err := sk.Sign(seed, dummy) + assert.Error(t, err) + assert.True(t, IsInvalidHasherSizeError(err)) + _, err = sk.PublicKey().Verify(sig, seed, dummy) + assert.Error(t, err) + assert.True(t, IsInvalidHasherSizeError(err)) + }) } } @@ -524,3 +544,154 @@ func TestECDSAHighAndLowS(t *testing.T) { } }) } + +// Test function only to flip S in a signature. It is used for testing signature malleability +func (a *ecdsaContext) signatureFlipS(sig []byte) []byte { + // read S + nLen := bitsToBytes(a.curveN.BitLen()) + s := new(big.Int).SetBytes(sig[nLen:]) + // compute N-S + sComplement := new(big.Int).Sub(a.curveN, s) + // write it into a new signature + newSig := make([]byte, len(sig)) + copy(newSig, sig[:nLen]) // copy R + sComplement.FillBytes(newSig[nLen:]) // write S complement + return newSig +} + +// TestECDSASecp256k1DeterministicSigning checks deterministic ECDSA signatures +// on secp256k1 against RFC 6979 known-answer test vectors. +// The vectors are the community secp256k1/SHA-256 vectors +// replicated in trezor-crypto and python-ecdsa. +// The expected signatures are the low-S normalized (r || s) pairs. +func TestECDSASecp256k1DeterministicSigning(t *testing.T) { + vectors := []struct { + sk string + msg string + sig string + }{ + { + sk: "0000000000000000000000000000000000000000000000000000000000000001", + msg: "Satoshi Nakamoto", + sig: "934b1ea10a4b3c1757e2b0c017d0b6143ce3c9a7e6a4a49860d7a6ab210ee3d82442ce9d2b916064108014783e923ec36b49743e2ffa1c4496f01a512aafd9e5", + }, + { + // the private key is the curve order minus 1 + sk: "fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364140", + msg: "Satoshi Nakamoto", + sig: "fd567d121db66e382991534ada77a6bd3106f0a1098c231e47993447cd6af2d06b39cd0eb1bc8603e159ef5c20a5c8ad685a45b06ce9bebed3f153d10d93bed5", + }, + { + sk: "f8b8af8ce3c7cca5e300d33939540c10d45ce001b8f252bfbc57ba0342904181", + msg: "Alan Turing", + sig: "7063ae83e7f62bbb171798131b4a0564b956930092b33b07b395615d9ec7e15c58dfcc1e00a35e1572f366ffe34ba0fc47db1e7189759b9fb233c5b05ab388ea", + }, + } + + for i, v := range vectors { + skBytes, err := hex.DecodeString(v.sk) + require.NoError(t, err) + sk, err := DecodePrivateKey(ECDSASecp256k1, skBytes) + require.NoError(t, err) + + sig, err := sk.Sign([]byte(v.msg), hash.NewSHA2_256()) + require.NoError(t, err) + assert.Equal(t, v.sig, hex.EncodeToString(sig), "vector %d", i) + + // the signature must verify under the matching public key + valid, err := sk.PublicKey().Verify(sig, []byte(v.msg), hash.NewSHA2_256()) + require.NoError(t, err) + assert.True(t, valid, "vector %d", i) + } +} + +// TestECDSAConcurrentPublicKey checks that concurrent calls to PublicKey +// on the same private key are safe and return equal keys. +// The test is effective when the race detector is enabled. +func TestECDSAConcurrentPublicKey(t *testing.T) { + for _, curve := range ecdsaCurves { + t.Run(curve.String(), func(t *testing.T) { + seed := make([]byte, KeyGenSeedMinLen) + _, err := crand.Read(seed) + require.NoError(t, err) + sk, err := GeneratePrivateKey(curve, seed) + require.NoError(t, err) + + pks := make([]PublicKey, 10) + var wg sync.WaitGroup + for i := range pks { + wg.Add(1) + go func() { + defer wg.Done() + pks[i] = sk.PublicKey() + }() + } + wg.Wait() + + for _, pk := range pks { + require.NotNil(t, pk) + assert.True(t, pk.Equals(pks[0])) + } + }) + } +} + +// TestECDSANilChecks covers the nil-related edge cases of the public API: +// decoding errors must return untyped nil interfaces, +// and Equals with a nil input must return false instead of panicking. +func TestECDSANilChecks(t *testing.T) { + t.Run("decode error paths return untyped nil interfaces", func(t *testing.T) { + for _, curve := range ecdsaCurves { + sk, err := DecodePrivateKey(curve, make([]byte, ecdsaPrKeyLen[curve]-1)) + require.Error(t, err) + // `assert.Nil` treats a typed-nil pointer inside an interface as nil, + // so compare against nil directly instead + assert.True(t, sk == nil) + + pk, err := DecodePublicKey(curve, make([]byte, ecdsaPubKeyLen[curve]-1)) + require.Error(t, err) + assert.True(t, pk == nil) + + pk, err = DecodePublicKeyCompressed(curve, make([]byte, ecdsaPubKeyLen[curve]-1)) + require.Error(t, err) + assert.True(t, pk == nil) + } + }) + + t.Run("Equals with a nil input returns false", func(t *testing.T) { + for _, curve := range ecdsaCurves { + seed := make([]byte, KeyGenSeedMinLen) + _, err := crand.Read(seed) + require.NoError(t, err) + sk, err := GeneratePrivateKey(curve, seed) + require.NoError(t, err) + assert.False(t, sk.Equals(nil)) + assert.False(t, sk.PublicKey().Equals(nil)) + } + }) +} + +// TestECDSASecp256k1CompressedDecoding checks compressed point decoding on secp256k1 +// using edge-case points where a generic (crypto/elliptic style) decompression +// either fails or computes a square root that doesn't match secp256k1 arithmetic. +func TestECDSASecp256k1CompressedDecoding(t *testing.T) { + testVectors := []string{ + "028b10bf56476bf7da39a3286e29df389177a2fa0fca2d73348ff78887515d8da1", // IsOnCurve for elliptic returns false + "03d39427f07f680d202fe8504306eb29041aceaf4b628c2c69b0ec248155443166", // odd, IsOnCurve for elliptic returns false + "0267d1942a6cbe4daec242ea7e01c6cdb82dadb6e7077092deb55c845bf851433e", // arith of sqrt in elliptic doesn't match secp256k1 + "0345d45eda6d087918b041453a96303b78c478dce89a4ae9b3c933a018888c5e06", // odd, arith of sqrt in elliptic doesn't match secp256k1 + } + + for _, testVector := range testVectors { + // get the compressed bytes + publicBytes, err := hex.DecodeString(testVector) + require.NoError(t, err) + + // decompress, check that those are perfectly valid secp256k1 public keys + retrieved, err := DecodePublicKeyCompressed(ECDSASecp256k1, publicBytes) + require.NoError(t, err) + + // check the compression is canonical by re-compressing to the same bytes + require.Equal(t, retrieved.EncodeCompressed(), publicBytes) + } +} From b44ebccbcf46f989d52ff28faefee28e8a5c18e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 11 Aug 2026 11:35:39 -0700 Subject: [PATCH 19/21] address review comments - build the public key eagerly at private key construction instead of lazily in PublicKey(), which removes the concurrency guards - read the secp256k1 curve constants from go-ethereum instead of hardcoding them and cross-checking - check the hasher output in bytes only, based on len(h), and drop the redundant hasher.Size() check - drop the P-256 public key length check, already covered by ParseUncompressedPublicKey - return the untyped nil once after the switch in the decoding functions - move the nil-input checks to sign_test_utils.go so they cover all algos, and extend them to DecodePublicKeyCompressed - document that the RFC 6979 vectors only test the current implementation, not a property the package guarantees - drop the concurrent PublicKey and compressed edge point tests --- ecdsa.go | 93 ++++++++++++++++++----------------------- ecdsa_p256.go | 27 ++++-------- ecdsa_secp256k1.go | 66 +++++++++++------------------ ecdsa_test.go | 101 ++++----------------------------------------- sign_test_utils.go | 48 +++++++++++---------- 5 files changed, 108 insertions(+), 227 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index ffbcfffd..b7d2c848 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -65,18 +65,15 @@ func (a *ecdsaContext) checkHasherAndComputeHash(msg []byte, hasher hash.Hasher) return nil, errNilHasher } - // check hasher's size is at least the curve order in bytes - nLen := bitsToBytes((a.curveN).BitLen()) - if hasher.Size() < nLen { - return nil, invalidHasherSizeErrorf( - "hasher's size should be at least %d bytes, got %d bytes", nLen, hasher.Size()) - } - h := hasher.ComputeHash(msg) - // guard against hasher implementations that compute fewer bytes - // than their declared size, - // since callers truncate the hash to the curve order size - // and would panic on a shorter slice + // check the computed hash is at least the curve order in bytes. + // All curve orders supported by the package have a bit-length multiple of 8, + // so callers truncate the message hash in bytes + // and the check is done in bytes too. + // The check uses the computed hash length rather than the hasher's declared size, + // so that a hasher implementation computing fewer bytes than it declares + // is rejected instead of panicking in the caller's truncation. + nLen := bitsToBytes((a.curveN).BitLen()) if len(h) < nLen { return nil, invalidHasherSizeErrorf( "hasher's output should be at least %d bytes, got %d bytes", nLen, len(h)) @@ -146,21 +143,23 @@ func (a *ecdsaContext) privateKey(d *big.Int) (PrivateKey, error) { d.FillBytes(dBytes) // dBytes is the big-endian encoding of d padded to the curve order // build the private key depending on the curve + var sk PrivateKey + var err error switch a.algo { case ECDSAP256: - sk, err := privateKeyECDSAP256(a, dBytes) - if err != nil { - // return an untyped nil, - // otherwise the returned interface is non-nil - // although it holds a nil pointer - return nil, err - } - return sk, nil + sk, err = privateKeyECDSAP256(a, dBytes) case ECDSASecp256k1: - return privateKeyECDSASecp256k1(a, dBytes), nil + sk = privateKeyECDSASecp256k1(a, dBytes) default: return nil, invalidInputsErrorf("the curve is not supported") } + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return sk, nil } // generatePrivateKey generates a private key for ECDSA @@ -235,28 +234,23 @@ func (a *ecdsaContext) decodePrivateKey(der []byte) (PrivateKey, error) { // Error Returns: // - invalidInputsError if the input is not a valid serialization of a public key on the given curve. func (a *ecdsaContext) rawDecodePublicKey(input []byte) (PublicKey, error) { + var pk PublicKey + var err error switch a.algo { case ECDSAP256: - pk, err := publicKeyECDSAP256(input) - if err != nil { - // return an untyped nil, - // otherwise the returned interface is non-nil - // although it holds a nil pointer - return nil, err - } - return pk, nil + pk, err = publicKeyECDSAP256(input) case ECDSASecp256k1: - pk, err := publicKeyECDSASecp256k1(a, input) - if err != nil { - // return an untyped nil, - // otherwise the returned interface is non-nil - // although it holds a nil pointer - return nil, err - } - return pk, nil + pk, err = publicKeyECDSASecp256k1(a, input) default: return nil, invalidInputsErrorf("curve is not supported") } + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil } func (a *ecdsaContext) decodePublicKey(der []byte) (PublicKey, error) { @@ -271,28 +265,23 @@ func (a *ecdsaContext) decodePublicKey(der []byte) (PublicKey, error) { // - invalidInputsError if the curve isn't supported or the input isn't a valid key serialization // on the given curve. func (a *ecdsaContext) decodePublicKeyCompressed(pkBytes []byte) (PublicKey, error) { + var pk PublicKey + var err error switch a.algo { case ECDSAP256: - pk, err := p256DecodePublicKeyCompressed(pkBytes) - if err != nil { - // return an untyped nil, - // otherwise the returned interface is non-nil - // although it holds a nil pointer - return nil, err - } - return pk, nil + pk, err = p256DecodePublicKeyCompressed(pkBytes) case ECDSASecp256k1: - pk, err := secp256k1DecodePublicKeyCompressed(pkBytes) - if err != nil { - // return an untyped nil, - // otherwise the returned interface is non-nil - // although it holds a nil pointer - return nil, err - } - return pk, nil + pk, err = secp256k1DecodePublicKeyCompressed(pkBytes) default: return nil, invalidInputsErrorf("the input curve is not supported") } + if err != nil { + // return an untyped nil, + // otherwise the returned interface is non-nil + // although it holds a nil pointer + return nil, err + } + return pk, nil } // Algorithm returns the algo related to the private key diff --git a/ecdsa_p256.go b/ecdsa_p256.go index 2f218277..8f6ade9e 100644 --- a/ecdsa_p256.go +++ b/ecdsa_p256.go @@ -24,7 +24,6 @@ import ( "crypto/rand" "fmt" "math/big" - "sync" "github.com/onflow/crypto/hash" ) @@ -66,9 +65,6 @@ type prKeyECDSAP256 struct { *prKeyCommonECDSA // go ecdsa standard lib private key goPrKey *ecdsa.PrivateKey - // pubKeyOnce guards the lazy construction of pubKey, - // making concurrent calls to PublicKey safe - pubKeyOnce sync.Once // public key pubKey *pubKeyECDSAP256 } @@ -94,7 +90,10 @@ func privateKeyECDSAP256(a *ecdsaContext, dBytes []byte) (*prKeyECDSAP256, error sk := &prKeyECDSAP256{ prKeyCommonECDSA: &prKeyCommonECDSA{a}, goPrKey: internalSK, - pubKey: nil, // public key is not constructed yet + pubKey: &pubKeyECDSAP256{ + pubKeyCommonECDSA: &pubKeyCommonECDSA{p256Instance}, + goPubKey: &internalSK.PublicKey, + }, } return sk, nil } @@ -133,20 +132,17 @@ func (sk *prKeyECDSAP256) String() string { // returns a publicKeyECDSAP256 from (bytes(x) || bytes(y)) bytes func publicKeyECDSAP256(XYBytes []byte) (*pubKeyECDSAP256, error) { - if len(XYBytes) != 2*pLenP256 { - return nil, invalidInputsErrorf("input has incorrect %s key size, got %d, expects %d", - ECDSAP256, len(XYBytes), 2*pLenP256) - } - // deserialization uses SEC1 version 2 (https://www.secg.org/sec1-v2.pdf section 2.3.3) // and includes on curve check. // The bytes serialization for non-infinity points is `0x04 || X || Y` and infinity point should be rejected anyway parsingBytes := append([]byte{ecEncodingUncompressed}, XYBytes...) - // ParseUncompressedPublicKey includes x

Date: Tue, 11 Aug 2026 14:06:23 -0500 Subject: [PATCH 20/21] keep TestECDSASecp256k1DeterministicSigning strictly about signing determinism --- ecdsa_test.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ecdsa_test.go b/ecdsa_test.go index 91531b72..7fa3daa6 100644 --- a/ecdsa_test.go +++ b/ecdsa_test.go @@ -558,11 +558,11 @@ func (a *ecdsaContext) signatureFlipS(sig []byte) []byte { return newSig } -// TestECDSASecp256k1DeterministicSigning checks deterministic ECDSA signatures -// on secp256k1 against RFC 6979 known-answer test vectors. +// TestECDSASecp256k1DeterministicSigning checks the current ECDSA signatures +// on secp256k1 against RFC 6979 known test vectors. // The vectors are the community secp256k1/SHA-256 vectors // replicated in trezor-crypto and python-ecdsa. -// The expected signatures are the low-S normalized (r || s) pairs. +// The expected signatures are low-S normalized. // // The test only makes sense while the underlying implementation (currently go-ethereum) // uses RFC 6979 nonces and outputs low-S signatures. @@ -605,10 +605,5 @@ func TestECDSASecp256k1DeterministicSigning(t *testing.T) { sig, err := sk.Sign([]byte(v.msg), hash.NewSHA2_256()) require.NoError(t, err) assert.Equal(t, v.sig, hex.EncodeToString(sig), "vector %d", i) - - // the signature must verify under the matching public key - valid, err := sk.PublicKey().Verify(sig, []byte(v.msg), hash.NewSHA2_256()) - require.NoError(t, err) - assert.True(t, valid, "vector %d", i) } } From 28f8bcadcf5e31519fdc0bd783af97a1faafa0fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Tue, 11 Aug 2026 12:15:24 -0700 Subject: [PATCH 21/21] address PR feedback --- ecdsa.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ecdsa.go b/ecdsa.go index b7d2c848..a7bf466d 100644 --- a/ecdsa.go +++ b/ecdsa.go @@ -33,7 +33,8 @@ import ( "github.com/onflow/crypto/hash" ) -// ecdsaContext embeds SigningAlgorithm +// ecdsaContext holds the signing algorithm and the curve parameters +// shared by all ECDSA keys on that curve. type ecdsaContext struct { // the signing algo algo SigningAlgorithm @@ -335,7 +336,7 @@ func (pk *pubKeyCommonECDSA) Size() int { return 2 * bitsToBytes(pk.curveP.BitLen()) } -// Equals tests the equality of two private keys +// Equals tests the equality of two public keys func pubKeyCommonECDSAEquals(pk, other PublicKey) bool { // a nil key is not equal to any key if other == nil {