From ecc0a4b760170f84b6d086499db21e36ba87ef6c Mon Sep 17 00:00:00 2001 From: Daniele Lacamera Date: Fri, 6 Nov 2026 13:34:57 +0100 Subject: [PATCH] Guard cpuid dispatch reads against the all-ones sentinel The file-local cpuid_flags statics in sha3.c, wc_frodokem_mat.c, wc_mlkem_poly.c and wc_mldsa.c start out as WC_CPUID_INITIALIZER, which is all ones. Any dispatch read before cpuid_get_flags_ex() therefore reports every feature present and sends a CPU without the extension into the asm path, where it dies with SIGILL. The dispatchers assume a prior key init set the flags, but the unit tests and white-box harnesses call them directly, without wc_MlDsaKey_Init() (and friends). On a CPU without AVX-512 this crashed the ML-DSA white-box harness in wc_mldsa_make_hint_88_avx512. aes.c already carries this pattern: a lazy init check before the dispatch. Add the same guard here - every dispatch now reads the flags through a per-file helper that resolves the sentinel on first use, so a read before any init reports no features and takes the C fallback instead of trapping. Verification: configure --enable-intelasm, full testwolfcrypt KAT suite passes (ML-DSA, ML-KEM, FrodoKEM, SHA-3), unit test groups mldsa/mlkem/frodokem/sha3 all pass; the ML-DSA AVX2 white-box harness that SIGILL'd before now runs to completion. --- wolfcrypt/src/sha3.c | 15 ++- wolfcrypt/src/wc_frodokem_mat.c | 95 +++++++------- wolfcrypt/src/wc_mldsa.c | 219 +++++++++++++++++--------------- wolfcrypt/src/wc_mlkem_poly.c | 189 ++++++++++++++------------- 4 files changed, 275 insertions(+), 243 deletions(-) diff --git a/wolfcrypt/src/sha3.c b/wolfcrypt/src/sha3.c index 7b2a6620a0c..dca554c1a0a 100644 --- a/wolfcrypt/src/sha3.c +++ b/wolfcrypt/src/sha3.c @@ -121,6 +121,13 @@ #include static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; + + static WC_INLINE cpuid_flags_t sha3_cpuid_flags(void) + { + if (cpuid_flags == WC_CPUID_INITIALIZER) + cpuid_get_flags_ex(&cpuid_flags); + return cpuid_flags; + } #ifdef WC_C_DYNAMIC_FALLBACK #define SHA3_BLOCK (sha3->sha3_block) #define SHA3_BLOCK_N (sha3->sha3_block_n) @@ -951,7 +958,7 @@ static int InitSha3(wc_Sha3* sha3) #endif #if defined(WOLFSSL_SHA3_AVX2) && !defined(WOLFSSL_SHA3_NO_AVX2) /* WOLFSSL_SHA3_AVX2 asks for AVX2 ahead of BMI2. */ - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(sha3_cpuid_flags())) { SHA3_BLOCK = sha3_block_avx2; SHA3_BLOCK_N = sha3_block_n_avx2; } @@ -959,14 +966,14 @@ static int InitSha3(wc_Sha3* sha3) #endif /* BMI2 before AVX2: sha3_block_bmi2 uses general registers only, so * it needs no vector-register save. */ - if (IS_INTEL_BMI1(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI1(sha3_cpuid_flags()) && IS_INTEL_BMI2(sha3_cpuid_flags())) { SHA3_BLOCK = sha3_block_bmi2; SHA3_BLOCK_N = sha3_block_n_bmi2; } #if !defined(WOLFSSL_SHA3_AVX2) && !defined(WOLFSSL_SHA3_NO_AVX2) /* AVX2 without BMI2. Only live in the plain build: the overrides * either select AVX2 above or disable it outright. */ - else if (SHA3_USE_AVX2(cpuid_flags)) { + else if (SHA3_USE_AVX2(sha3_cpuid_flags())) { SHA3_BLOCK = sha3_block_avx2; SHA3_BLOCK_N = sha3_block_n_avx2; } @@ -985,7 +992,7 @@ static int InitSha3(wc_Sha3* sha3) } else #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(sha3_cpuid_flags())) { SHA3_BLOCK = BlockSha3_crypto; SHA3_BLOCK_N = NULL; } diff --git a/wolfcrypt/src/wc_frodokem_mat.c b/wolfcrypt/src/wc_frodokem_mat.c index 47335f010d0..4c78c446e01 100644 --- a/wolfcrypt/src/wc_frodokem_mat.c +++ b/wolfcrypt/src/wc_frodokem_mat.c @@ -94,10 +94,10 @@ #define FRODOKEM_SA_ACCUM(out, s, row, j, n) \ do { \ FRODOKEM_SA_ACCUM_SME(out, s, row, j, n) \ - if (IS_AARCH64_SVE(cpuid_flags)) { \ + if (IS_AARCH64_SVE(frodokem_mat_cpuid_flags())) { \ frodokem_sa_accum_sve((out), (s), (row), (j), (n)); \ } \ - else if (IS_AARCH64_ASIMD(cpuid_flags)) { \ + else if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { \ frodokem_sa_accum_arm((out), (s), (row), (j), (n)); \ } \ else { \ @@ -107,10 +107,10 @@ #define FRODOKEM_AS_ACCUM(out, s, row, i, n) \ do { \ FRODOKEM_AS_ACCUM_SME(out, s, row, i, n) \ - if (IS_AARCH64_SVE(cpuid_flags)) { \ + if (IS_AARCH64_SVE(frodokem_mat_cpuid_flags())) { \ frodokem_as_accum_sve((out), (s), (row), (i), (n)); \ } \ - else if (IS_AARCH64_ASIMD(cpuid_flags)) { \ + else if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { \ frodokem_as_accum_arm((out), (s), (row), (i), (n)); \ } \ else { \ @@ -121,7 +121,7 @@ #define FRODOKEM_SA_ACCUM(out, s, row, j, n) \ do { \ FRODOKEM_SA_ACCUM_SME(out, s, row, j, n) \ - if (IS_AARCH64_ASIMD(cpuid_flags)) { \ + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { \ frodokem_sa_accum_arm((out), (s), (row), (j), (n)); \ } \ else { \ @@ -131,7 +131,7 @@ #define FRODOKEM_AS_ACCUM(out, s, row, i, n) \ do { \ FRODOKEM_AS_ACCUM_SME(out, s, row, i, n) \ - if (IS_AARCH64_ASIMD(cpuid_flags)) { \ + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { \ frodokem_as_accum_arm((out), (s), (row), (i), (n)); \ } \ else { \ @@ -191,6 +191,13 @@ * matrix-A generation on Intel, and the SVE / NEON matrix ops on AArch64 (NEON * is gated on Advanced SIMD). Populated once by frodokem_init(). */ static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; + +static WC_INLINE cpuid_flags_t frodokem_mat_cpuid_flags(void) +{ + if (cpuid_flags == WC_CPUID_INITIALIZER) + cpuid_get_flags_ex(&cpuid_flags); + return cpuid_flags; +} #endif #if defined(FRODOKEM_HAVE_SME) && !defined(WOLF_CRYPTO_CB_ONLY_FRODOKEM) @@ -226,7 +233,7 @@ void frodokem_init(void) #endif #if defined(FRODOKEM_HAVE_SME) && !defined(WOLF_CRYPTO_CB_ONLY_FRODOKEM) /* SME needs SVL >= 256 bits (>= 32 bytes) for the 8 x 8 ZA.S tile. */ - frodokem_sme_svl_ok = IS_AARCH64_SME(cpuid_flags) && + frodokem_sme_svl_ok = IS_AARCH64_SME(frodokem_mat_cpuid_flags()) && (frodokem_sme_svl_bytes() >= 32); #endif } @@ -929,20 +936,20 @@ static void frodokem_sample_matrix(word16* mat, int cnt, const byte* r, frodokem_sample_neon(mat, cnt, p->cdf, p->cdfLen); #else #ifdef FRODOKEM_HAVE_NEON_RUNTIME - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { frodokem_sample_neon(mat, cnt, p->cdf, p->cdfLen); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_sample_avx512(mat, cnt, p->cdf, p->cdfLen); RESTORE_VECTOR_REGISTERS(); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_sample_avx2(mat, cnt, p->cdf, p->cdfLen); RESTORE_VECTOR_REGISTERS(); } @@ -1509,7 +1516,7 @@ static int frodokem_gen_a_row_shake_x2(word16* rows, const byte* seedA, } #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 /* Use the SHA3 crypto extension (EOR3/RAX1/XAR/BCAX) when present. */ - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(frodokem_mat_cpuid_flags())) { frodokem_sha3_x2_crypto(state); } else @@ -1735,7 +1742,7 @@ static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s, /* Generate the A rows in batches matching the fused asm accumulate (eight * with AVX512, four with AVX2); n is a multiple of 8 for every set. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if ((ret == 0) && USE_INTEL_AVX512(cpuid_flags) && + if ((ret == 0) && USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { /* The AES-NI/VAES row kernels below consume aes->key directly, which * holds a valid AES-NI-layout key schedule only when the @@ -1745,7 +1752,7 @@ static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s, * only the C-fallback schedule (aes->key_C_fallback) - re-key under * the held region, where the nested SAVE_VECTOR_REGISTERS2() always * succeeds, so aes->key is valid for the kernels. */ - if (IS_INTEL_AESNI(cpuid_flags) && (! aes->use_aesni)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags()) && (! aes->use_aesni)) { ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL, AES_ENCRYPTION); } @@ -1754,13 +1761,13 @@ static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s, * VAES (whole batch in one asm call), else the AES-NI register * kernel, else the per-row C generator. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES - if (IS_INTEL_VAES(cpuid_flags)) { + if (IS_INTEL_VAES(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_avx512((byte*)row, row, (const byte*)aes->key, i, 8, n, (int)p->qMask); } else #endif - if (IS_INTEL_AESNI(cpuid_flags)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_aesni((byte*)row, row, (const byte*)aes->key, i, 8, n, (int)p->qMask); } @@ -1777,11 +1784,11 @@ static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s, else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if ((ret == 0) && IS_INTEL_AVX2(cpuid_flags) && + if ((ret == 0) && IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { /* Re-key if SetKey lacked vector registers - see the note in the * AVX512 branch above (or in frodokem_mul_add_as_plus_e_aes). */ - if (IS_INTEL_AESNI(cpuid_flags) && (! aes->use_aesni)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags()) && (! aes->use_aesni)) { ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL, AES_ENCRYPTION); } @@ -1790,13 +1797,13 @@ static int frodokem_mul_add_as_plus_e_aes(word16* out, const word16* s, * VAES (whole batch in one asm call), else the AES-NI register * kernel, else the per-row C generator. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES - if (IS_INTEL_VAES(cpuid_flags)) { + if (IS_INTEL_VAES(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_avx2((byte*)row, row, (const byte*)aes->key, i, 4, n, (int)p->qMask); } else #endif - if (IS_INTEL_AESNI(cpuid_flags)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_aesni((byte*)row, row, (const byte*)aes->key, i, 4, n, (int)p->qMask); } @@ -1883,7 +1890,7 @@ static int frodokem_mul_add_as_plus_e_shake(word16* out, const word16* s, /* When AVX512 is available, generate and consume eight A rows at a time * with the 8-way SHAKE permutation. n is a multiple of 8 for every param * set, so there is no remainder loop. */ - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { ret = 0; for (i = 0; i < n; i += 8) { ret = frodokem_gen_a_row_shake_x8(row, seedA, i, p); @@ -1901,7 +1908,7 @@ static int frodokem_mul_add_as_plus_e_shake(word16* out, const word16* s, /* When AVX2 is available, generate and consume four A rows at a time with * the 4-way SHAKE permutation. n is a multiple of 4 for every parameter * set, so there is no remainder loop. */ - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { ret = 0; for (i = 0; i < n; i += 4) { ret = frodokem_gen_a_row_shake_x4(row, seedA, i, p); @@ -1920,7 +1927,7 @@ static int frodokem_mul_add_as_plus_e_shake(word16* out, const word16* s, * with the 2-way NEON SHAKE; without it, fall back to the scalar SHAKE and * portable-C accumulate (the 2-way permute is itself NEON). n is a multiple * of 2 for every set. */ - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { (void)shake; ret = 0; for (i = 0; i < n; i += 2) { @@ -2057,7 +2064,7 @@ static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s, /* Generate the A rows in batches matching the fused asm accumulate (eight * with AVX512, four with AVX2); n is a multiple of 8 for every set. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if ((ret == 0) && USE_INTEL_AVX512(cpuid_flags) && + if ((ret == 0) && USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { /* The AES-NI/VAES row kernels below consume aes->key directly, which * holds a valid AES-NI-layout key schedule only when the @@ -2067,7 +2074,7 @@ static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s, * only the C-fallback schedule (aes->key_C_fallback) - re-key under * the held region, where the nested SAVE_VECTOR_REGISTERS2() always * succeeds, so aes->key is valid for the kernels. */ - if (IS_INTEL_AESNI(cpuid_flags) && (! aes->use_aesni)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags()) && (! aes->use_aesni)) { ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL, AES_ENCRYPTION); } @@ -2076,13 +2083,13 @@ static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s, * VAES (whole batch in one asm call), else the AES-NI register * kernel, else the per-row C generator. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES - if (IS_INTEL_VAES(cpuid_flags)) { + if (IS_INTEL_VAES(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_avx512((byte*)row, row, (const byte*)aes->key, j, 8, n, (int)p->qMask); } else #endif - if (IS_INTEL_AESNI(cpuid_flags)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_aesni((byte*)row, row, (const byte*)aes->key, j, 8, n, (int)p->qMask); } @@ -2099,11 +2106,11 @@ static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s, else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if ((ret == 0) && IS_INTEL_AVX2(cpuid_flags) && + if ((ret == 0) && IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { /* Re-key if SetKey lacked vector registers - see the note in the * AVX512 branch above (or in frodokem_mul_add_as_plus_e_aes). */ - if (IS_INTEL_AESNI(cpuid_flags) && (! aes->use_aesni)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags()) && (! aes->use_aesni)) { ret = wc_AesSetKeyDirect(aes, seedA, FRODOKEM_SEEDA_SZ, NULL, AES_ENCRYPTION); } @@ -2112,13 +2119,13 @@ static int frodokem_mul_add_sa_plus_e_aes(word16* out, const word16* s, * VAES (whole batch in one asm call), else the AES-NI register * kernel, else the per-row C generator. */ #ifdef FRODOKEM_HAVE_MATRIX_ASM_VAES - if (IS_INTEL_VAES(cpuid_flags)) { + if (IS_INTEL_VAES(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_avx2((byte*)row, row, (const byte*)aes->key, j, 4, n, (int)p->qMask); } else #endif - if (IS_INTEL_AESNI(cpuid_flags)) { + if (IS_INTEL_AESNI(frodokem_mat_cpuid_flags())) { frodokem_gen_a_rows_aes_aesni((byte*)row, row, (const byte*)aes->key, j, 4, n, (int)p->qMask); } @@ -2209,7 +2216,7 @@ static int frodokem_mul_add_sa_plus_e_shake(word16* out, const word16* s, /* When AVX512 is available, generate and consume eight A rows at a time * with the 8-way SHAKE permutation. n is a multiple of 8 for every param * set, so there is no remainder loop. */ - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { ret = 0; for (j = 0; j < n; j += 8) { ret = frodokem_gen_a_row_shake_x8(row, seedA, j, p); @@ -2227,7 +2234,7 @@ static int frodokem_mul_add_sa_plus_e_shake(word16* out, const word16* s, /* When AVX2 is available, generate and consume four A rows at a time with * the 4-way SHAKE permutation. n is a multiple of 4 for every parameter * set, so there is no remainder loop. */ - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { ret = 0; for (j = 0; j < n; j += 4) { ret = frodokem_gen_a_row_shake_x4(row, seedA, j, p); @@ -2246,7 +2253,7 @@ static int frodokem_mul_add_sa_plus_e_shake(word16* out, const word16* s, * with the 2-way NEON SHAKE (SVE or NEON accumulate); without it, fall back * to the scalar SHAKE and portable-C accumulate (the 2-way permute is * itself NEON). n is a multiple of 2 for every set. */ - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { (void)shake; ret = 0; for (j = 0; j < n; j += 2) { @@ -2369,20 +2376,20 @@ void frodokem_mul_add_sb_plus_e(word16* out, const word16* b, const word16* s, frodokem_mul_add_sb_plus_e_arm(out, b, s, n, qmask); #else #ifdef FRODOKEM_HAVE_NEON_RUNTIME - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { frodokem_mul_add_sb_plus_e_arm(out, b, s, n, qmask); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_mul_add_sb_plus_e_avx512(out, b, s, n, qmask); RESTORE_VECTOR_REGISTERS(); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_mul_add_sb_plus_e_avx2(out, b, s, n, qmask); RESTORE_VECTOR_REGISTERS(); } @@ -2641,26 +2648,26 @@ void frodokem_mul_bs(word16* out, const word16* b, const word16* s, frodokem_mul_bs_arm(out, b, s, n, qmask); #else #ifdef FRODOKEM_HAVE_SVE - if (IS_AARCH64_SVE(cpuid_flags)) { + if (IS_AARCH64_SVE(frodokem_mat_cpuid_flags())) { frodokem_mul_bs_sve(out, b, s, n, qmask); } else #endif #ifdef FRODOKEM_HAVE_NEON_RUNTIME - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { frodokem_mul_bs_arm(out, b, s, n, qmask); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_mul_bs_avx512(out, b, s, n, qmask); RESTORE_VECTOR_REGISTERS(); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_mul_bs_avx2(out, b, s, n, qmask); RESTORE_VECTOR_REGISTERS(); } @@ -2902,26 +2909,26 @@ void frodokem_add(word16* a, const word16* b, int qmask) frodokem_add_arm(a, b, qmask); #else #ifdef FRODOKEM_HAVE_SVE - if (IS_AARCH64_SVE(cpuid_flags)) { + if (IS_AARCH64_SVE(frodokem_mat_cpuid_flags())) { frodokem_add_sve(a, b, qmask); } else #endif #ifdef FRODOKEM_HAVE_NEON_RUNTIME - if (IS_AARCH64_ASIMD(cpuid_flags)) { + if (IS_AARCH64_ASIMD(frodokem_mat_cpuid_flags())) { frodokem_add_arm(a, b, qmask); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM_AVX512 - if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (USE_INTEL_AVX512(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_add_avx512(a, b, qmask); RESTORE_VECTOR_REGISTERS(); } else #endif #ifdef FRODOKEM_HAVE_MATRIX_ASM - if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { + if (IS_INTEL_AVX2(frodokem_mat_cpuid_flags()) && (SAVE_VECTOR_REGISTERS2() == 0)) { frodokem_add_avx2(a, b, qmask); RESTORE_VECTOR_REGISTERS(); } diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 00ec57819e3..1c6a27b1ee1 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -272,6 +272,19 @@ #if defined(USE_INTEL_SPEEDUP) || (defined(__aarch64__) && \ defined(WOLFSSL_ARMASM)) static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; + +/* cpuid_flags starts out as WC_CPUID_INITIALIZER, which is all ones: a + * dispatch read before cpuid_get_flags_ex() would report every feature + * present and send a CPU without the extension into the asm path, where it + * SIGILLs. The dispatchers can be reached without a prior init (the unit + * tests and white-box harnesses call them directly), so every dispatch reads + * the flags through this helper. */ +static WC_INLINE cpuid_flags_t mldsa_cpuid_flags(void) +{ + if (cpuid_flags == WC_CPUID_INITIALIZER) + cpuid_get_flags_ex(&cpuid_flags); + return cpuid_flags; +} #endif #if defined(USE_INTEL_SPEEDUP) @@ -317,7 +330,7 @@ static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH #define MLDSA_NEON_PICK(fn, ...) \ do { \ - if (IS_AARCH64_RDM(cpuid_flags)) { \ + if (IS_AARCH64_RDM(mldsa_cpuid_flags())) { \ fn##_sqrdmlsh_neon(__VA_ARGS__); \ } \ else { \ @@ -361,7 +374,7 @@ static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; !defined(WOLFSSL_MLDSA_NO_SIGN) static void mlkem_sha3_blocksx3(word64* state) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { sha3_blocksx3_crypto(state); } else { @@ -373,7 +386,7 @@ static void mlkem_sha3_blocksx3(word64* state) #ifdef MLDSA_NEON_EXPAND_A static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { sha3_128_blocksx3_seed_crypto(state, seed); } else { @@ -386,7 +399,7 @@ static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || !defined(WOLFSSL_MLDSA_NO_SIGN) static void mlkem_shake256_blocksx3_seed_64(word64* state, byte* seed) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mldsa_cpuid_flags())) { sha3_256_blocksx3_seed_64_crypto(state, seed); } else { @@ -817,14 +830,14 @@ static int mldsa_shake256(wc_Shake* shake256, const byte* data, dataLen -= WC_SHA3_256_COUNT * 8; data += WC_SHA3_256_COUNT * 8; #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -835,7 +848,7 @@ static int mldsa_shake256(wc_Shake* shake256, const byte* data, if (dataLen >= WC_SHA3_256_COUNT * 8) { #ifndef WC_SHA3_NO_ASM word32 n = dataLen / (WC_SHA3_256_COUNT * 8); - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -845,7 +858,7 @@ static int mldsa_shake256(wc_Shake* shake256, const byte* data, dataLen -= n; data += n; } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_n_bmi2(state, data, n, WC_SHA3_256_COUNT * 8); n *= WC_SHA3_256_COUNT * 8; dataLen -= n; @@ -876,14 +889,14 @@ static int mldsa_shake256(wc_Shake* shake256, const byte* data, } #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -992,14 +1005,14 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, data2Len -= WC_SHA3_256_COUNT * 8 - data1Len; data2 += WC_SHA3_256_COUNT * 8 - data1Len; #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -1011,7 +1024,7 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, if (data2Len >= WC_SHA3_256_COUNT * 8) { #ifndef WC_SHA3_NO_ASM word32 n = data2Len / (WC_SHA3_256_COUNT * 8); - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1021,7 +1034,7 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, data2Len -= n; data2 += n; } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_n_bmi2(state, data2, n, WC_SHA3_256_COUNT * 8); n *= WC_SHA3_256_COUNT * 8; data2Len -= n; @@ -1056,14 +1069,14 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, } #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -1444,14 +1457,14 @@ static int mldsa_squeeze256(wc_Shake* shake256, const byte* in, for (; outBlocks > 0; outBlocks--) { #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -1623,7 +1636,7 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, /* eta = 4 packs one polynomial per call - vpmovqb needs no lane pairing. * eta = 2 still pairs, so an odd trailing polynomial is left to the C * code: the AVX2 entry points take a whole vector, not one polynomial. */ - if (USE_INTEL_AVX512(cpuid_flags) && (eta == MLDSA_ETA_4)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags()) && (eta == MLDSA_ETA_4)) { unsigned int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -1636,7 +1649,7 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, } RESTORE_VECTOR_REGISTERS(); } - else if (USE_INTEL_AVX512(cpuid_flags) && (eta == MLDSA_ETA_2) && + else if (USE_INTEL_AVX512(mldsa_cpuid_flags()) && (eta == MLDSA_ETA_2) && ((d & 1) == 0)) { unsigned int i; unsigned int e = MLDSA_ETA_2_BITS * MLDSA_N / 8; @@ -1653,7 +1666,7 @@ static int mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1748,7 +1761,7 @@ static int mldsa_decode_eta_2_bits(const byte* p, sword32* s) } #else #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1836,7 +1849,7 @@ static int mldsa_decode_eta_4_bits(const byte* p, sword32* s) } #else #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1896,7 +1909,7 @@ static int mldsa_vec_decode_eta_bits(const byte* p, byte eta, sword32* s, i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Two polynomials per call; an odd trailing one falls through. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1924,7 +1937,7 @@ static int mldsa_vec_decode_eta_bits(const byte* p, byte eta, sword32* s, unsigned int e = MLDSA_N / 2; i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2094,7 +2107,7 @@ static int mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512_VBMI /* vpermb gathers a whole register's encoded bytes, so this needs no * polynomial pairing and takes any dimension. */ - if (IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mldsa_cpuid_flags())) { unsigned int i; unsigned int e0 = MLDSA_D * MLDSA_N / 8; unsigned int e1 = MLDSA_U * MLDSA_N / 8; @@ -2114,7 +2127,7 @@ static int mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, #endif /* Two polynomials per call; the AVX2 entry takes a whole vector, so an * odd dimension stays on it. */ - if (USE_INTEL_AVX512(cpuid_flags) && ((d & 1) == 0)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags()) && ((d & 1) == 0)) { unsigned int i; unsigned int e0 = MLDSA_D * MLDSA_N / 8; unsigned int e1 = MLDSA_U * MLDSA_N / 8; @@ -2133,7 +2146,7 @@ static int mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2248,7 +2261,7 @@ static int mldsa_decode_t0(const byte* t0, sword32* t) } #else #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2290,7 +2303,7 @@ static int mldsa_vec_decode_t0(const byte* t0, byte d, sword32* t) i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Two polynomials per call; an odd trailing one falls through. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2416,7 +2429,7 @@ static int mldsa_decode_t1(const byte* t1, sword32* t) } #else #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2458,7 +2471,7 @@ static int mldsa_vec_decode_t1(const byte* t1, byte d, sword32* t) i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Two polynomials per call; an odd trailing one falls through. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2553,7 +2566,7 @@ static int mldsa_encode_gamma1_17_bits(const sword32* z, byte* s) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2563,7 +2576,7 @@ static int mldsa_encode_gamma1_17_bits(const sword32* z, byte* s) else #endif #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2650,7 +2663,7 @@ static int mldsa_encode_gamma1_19_bits(const sword32* z, byte* s) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2660,7 +2673,7 @@ static int mldsa_encode_gamma1_19_bits(const sword32* z, byte* s) else #endif #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3035,7 +3048,7 @@ static int mldsa_decode_gamma1(const byte* s, int bits, sword32* z) } #else #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3086,7 +3099,7 @@ static int mldsa_vec_decode_gamma1(const byte* x, byte l, int bits, i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Two polynomials per call; an odd trailing one falls through. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3201,7 +3214,7 @@ static int mldsa_encode_w1_88(const sword32* w1, byte* w1e) #ifdef USE_INTEL_SPEEDUP /* Tests call this without wc_MlDsaKey_Init. */ cpuid_get_flags_ex(&cpuid_flags); - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3298,7 +3311,7 @@ static int mldsa_encode_w1_32(const sword32* w1, byte* w1e) #ifdef USE_INTEL_SPEEDUP /* Tests call this without wc_MlDsaKey_Init. */ cpuid_get_flags_ex(&cpuid_flags); - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3401,7 +3414,7 @@ static int mldsa_vec_encode_w1(const sword32* w1, byte k, sword32 gamma2, defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512_VBMI) /* vpermb gathers the encoded bytes of a whole register, so the VBMI * encoder needs no polynomial pairing and does the entire vector. */ - if (IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3415,7 +3428,7 @@ static int mldsa_vec_encode_w1(const sword32* w1, byte k, sword32 gamma2, #endif #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Two polynomials per pass; an odd trailing one falls through. */ - if ((i == 0) && USE_INTEL_AVX512(cpuid_flags)) { + if ((i == 0) && USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -3444,7 +3457,7 @@ static int mldsa_vec_encode_w1(const sword32* w1, byte k, sword32 gamma2, i = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* One polynomial per pass - vpmovqb needs no lane pairing. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4655,7 +4668,7 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Eight SHAKE-128 instances per permutation instead of four. Handles any * k x l, so it comes before the fixed-size AVX2 implementations. */ - if (USE_INTEL_AVX512(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags()) && IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4666,8 +4679,8 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, else #endif #ifndef WOLFSSL_NO_ML_DSA_44 - if ((k == 4) && (l == 4) && IS_INTEL_AVX2(cpuid_flags) && - IS_INTEL_BMI2(cpuid_flags)) { + if ((k == 4) && (l == 4) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && + IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4678,8 +4691,8 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, else #endif #ifndef WOLFSSL_NO_ML_DSA_65 - if ((k == 6) && (l == 5) && IS_INTEL_AVX2(cpuid_flags) && - IS_INTEL_BMI2(cpuid_flags)) { + if ((k == 6) && (l == 5) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && + IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4690,8 +4703,8 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, else #endif #ifndef WOLFSSL_NO_ML_DSA_87 - if ((k == 8) && (l == 7) && IS_INTEL_AVX2(cpuid_flags) && - IS_INTEL_BMI2(cpuid_flags)) { + if ((k == 8) && (l == 7) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && + IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6134,7 +6147,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Eight SHAKE-256 instances per permutation instead of four. Handles any * vector dimensions, so it comes before the fixed-size AVX2 code. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6145,7 +6158,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, else #endif #ifndef WOLFSSL_NO_ML_DSA_44 - if ((s1Len == 4) && IS_INTEL_AVX2(cpuid_flags)) { + if ((s1Len == 4) && IS_INTEL_AVX2(mldsa_cpuid_flags())) { sword32* s[2] = { s1, s2 }; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -6157,7 +6170,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, else #endif #ifndef WOLFSSL_NO_ML_DSA_65 - if ((s1Len == 5) && IS_INTEL_AVX2(cpuid_flags)) { + if ((s1Len == 5) && IS_INTEL_AVX2(mldsa_cpuid_flags())) { sword32* s[2] = { s1, s2 }; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -6169,7 +6182,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, else #endif #ifndef WOLFSSL_NO_ML_DSA_87 - if ((s1Len == 7) && IS_INTEL_AVX2(cpuid_flags)) { + if ((s1Len == 7) && IS_INTEL_AVX2(mldsa_cpuid_flags())) { sword32* s[2] = { s1, s2 }; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -6960,7 +6973,7 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, #elif defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Whole vector in one eight-way run, whatever the dimension. */ - if (USE_INTEL_AVX512(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags()) && IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6972,7 +6985,7 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, /* Each generator handles one dimension only, so test it before saving * the vector registers. */ #ifndef WOLFSSL_NO_ML_DSA_44 - if ((l == 4) && IS_INTEL_AVX2(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if ((l == 4) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6982,7 +6995,7 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, else #endif #ifndef WOLFSSL_NO_ML_DSA_65 - if ((l == 5) && IS_INTEL_AVX2(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if ((l == 5) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6992,7 +7005,7 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, else #endif #ifndef WOLFSSL_NO_ML_DSA_87 - if ((l == 7) && IS_INTEL_AVX2(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags)) { + if ((l == 7) && IS_INTEL_AVX2(mldsa_cpuid_flags()) && IS_INTEL_BMI2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7082,14 +7095,14 @@ static int mldsa_sample_in_ball_ex(int level, wc_Shake* shake256, if (k == MLDSA_GEN_C_BLOCK_BYTES) { /* Generate a new block. */ #ifndef WC_SHA3_NO_ASM - if (SHA3_USE_AVX2(cpuid_flags)) { + if (SHA3_USE_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; sha3_block_avx2(state); RESTORE_VECTOR_REGISTERS(); } - else if (IS_INTEL_BMI2(cpuid_flags)) { + else if (IS_INTEL_BMI2(mldsa_cpuid_flags())) { sha3_block_bmi2(state); } else @@ -7527,7 +7540,7 @@ static int mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7545,7 +7558,7 @@ static int mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7677,7 +7690,7 @@ static int mldsa_vec_check_low(const sword32* a, byte l, sword32 hi, #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { @@ -7690,7 +7703,7 @@ static int mldsa_vec_check_low(const sword32* a, byte l, sword32 hi, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { @@ -7787,7 +7800,7 @@ static int mldsa_make_hint_88(const sword32* s, const sword32* w1, byte* h, byte idx; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { @@ -7903,7 +7916,7 @@ static int mldsa_make_hint_32(const sword32* s, const sword32* w1, (void)omega; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { @@ -8387,7 +8400,7 @@ static int mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, if (gamma2 == MLDSA_Q_LOW_88) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -8396,7 +8409,7 @@ static int mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -8427,7 +8440,7 @@ static int mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, if (gamma2 == MLDSA_Q_LOW_32) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -8436,7 +8449,7 @@ static int mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9054,7 +9067,7 @@ static int mldsa_ntt(sword32* r) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9065,7 +9078,7 @@ static int mldsa_ntt(sword32* r) #endif #ifdef USE_INTEL_SPEEDUP /* MLDSA_NTT_AVX2: see the flavor-selection note by its definition. */ - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9116,7 +9129,7 @@ static int mldsa_ntt_full(sword32* r) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9126,7 +9139,7 @@ static int mldsa_ntt_full(sword32* r) else #endif #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9160,7 +9173,7 @@ static int mldsa_vec_ntt(sword32* r, byte l) #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* One polynomial per call, so there is no odd trailing polynomial to * hand back to mldsa_ntt() below. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9582,7 +9595,7 @@ static int mldsa_ntt_small(sword32* r) { int ret = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9594,7 +9607,7 @@ static int mldsa_ntt_small(sword32* r) #ifdef USE_INTEL_SPEEDUP /* MLDSA_NTT_SMALL_AVX2: see the flavor-selection note by its * definition. */ - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9627,7 +9640,7 @@ static int mldsa_ntt_small_full(sword32* r) { int ret = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -9637,7 +9650,7 @@ static int mldsa_ntt_small_full(sword32* r) else #endif #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10174,7 +10187,7 @@ static int mldsa_invntt(sword32* r) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10187,7 +10200,7 @@ static int mldsa_invntt(sword32* r) #endif #ifdef USE_INTEL_SPEEDUP /* MLDSA_INVNTT_AVX2: see the flavor-selection note by its definition. */ - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10266,7 +10279,7 @@ static int mldsa_invntt_full(sword32* r) } #else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10276,7 +10289,7 @@ static int mldsa_invntt_full(sword32* r) else #endif #ifdef USE_INTEL_SPEEDUP - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10311,7 +10324,7 @@ static int mldsa_vec_invntt_full(sword32* r, byte l) #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* One polynomial per call, so there is no odd trailing polynomial to * hand back to mldsa_invntt_full() below. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10539,7 +10552,7 @@ static int mldsa_matrix_mul(sword32* r, const sword32* m, const sword32* v, #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -10570,7 +10583,7 @@ static int mldsa_matrix_mul(sword32* r, const sword32* m, const sword32* v, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -10692,7 +10705,7 @@ static int mldsa_mul(sword32* r, sword32* a, sword32* b) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10701,7 +10714,7 @@ static int mldsa_mul(sword32* r, sword32* a, sword32* b) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10730,7 +10743,7 @@ static int mldsa_mul_invntt(sword32* r, sword32* c, sword32* v) int ret = 0; #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) /* Both steps under one save/restore of the vector registers. */ - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10806,7 +10819,7 @@ static int mldsa_vec_mul(sword32* r, sword32* a, sword32* b, byte l) #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10819,7 +10832,7 @@ static int mldsa_vec_mul(sword32* r, sword32* a, sword32* b, byte l) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10902,7 +10915,7 @@ static int mldsa_poly_red(sword32* a) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -10911,7 +10924,7 @@ static int mldsa_poly_red(sword32* a) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11033,7 +11046,7 @@ static int mldsa_sub(sword32* r, const sword32* a) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11042,7 +11055,7 @@ static int mldsa_sub(sword32* r, const sword32* a) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11131,7 +11144,7 @@ static int mldsa_add(sword32* r, const sword32* a) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11140,7 +11153,7 @@ static int mldsa_add(sword32* r, const sword32* a) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11233,7 +11246,7 @@ static int mldsa_make_pos(sword32* a) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -11242,7 +11255,7 @@ static int mldsa_make_pos(sword32* a) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mldsa_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -16333,11 +16346,9 @@ int wc_MlDsaKey_Init(wc_MlDsaKey* key, void* heap, int devId) #if defined(USE_INTEL_SPEEDUP) || (defined(__aarch64__) && \ defined(WOLFSSL_ARMASM)) /* AArch64 needs these for the FEAT_RDM check that picks the NTT flavor and - * the FEAT_SHA3 check that picks the Keccak one. This has to stay ahead of - * anything that can reach those dispatchers: cpuid_flags starts out as - * WC_CPUID_INITIALIZER, which is all ones, so a read before this call - * reports every feature present and would send a CPU without FEAT_SHA3 - * into the SHA-3 instruction path and SIGILL. */ + * the FEAT_SHA3 check that picks the Keccak one. The dispatchers are + * self-guarding (mldsa_cpuid_flags() resolves the all-ones sentinel), so + * this call is what turns the fast paths on from the first operation. */ cpuid_get_flags_ex(&cpuid_flags); #endif #if defined(MLDSA_SHAKE_DIRECT) diff --git a/wolfcrypt/src/wc_mlkem_poly.c b/wolfcrypt/src/wc_mlkem_poly.c index 2ab6159c187..b4ccfd6cbc3 100644 --- a/wolfcrypt/src/wc_mlkem_poly.c +++ b/wolfcrypt/src/wc_mlkem_poly.c @@ -140,6 +140,13 @@ extern sword16 wc_mlkem_opt_blocker(void); #if defined(USE_INTEL_SPEEDUP) || (defined(__aarch64__) && \ defined(WOLFSSL_ARMASM)) static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; + +static WC_INLINE cpuid_flags_t mlkem_poly_cpuid_flags(void) +{ + if (cpuid_flags == WC_CPUID_INITIALIZER) + cpuid_get_flags_ex(&cpuid_flags); + return cpuid_flags; +} #endif /* Half of Q plus one. Converted message bit value of 1. */ @@ -1309,7 +1316,7 @@ void mlkem_init(void) static void mlkem_sha3_blocksx3(word64* state) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mlkem_poly_cpuid_flags())) { sha3_blocksx3_crypto(state); } else { @@ -1319,7 +1326,7 @@ static void mlkem_sha3_blocksx3(word64* state) static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mlkem_poly_cpuid_flags())) { sha3_128_blocksx3_seed_crypto(state, seed); } else { @@ -1329,7 +1336,7 @@ static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) static void mlkem_shake256_blocksx3_seed(word64* state, byte* seed) { - if (IS_AARCH64_SHA3(cpuid_flags)) { + if (IS_AARCH64_SHA3(mlkem_poly_cpuid_flags())) { sha3_256_blocksx3_seed_crypto(state, seed); } else { @@ -1371,7 +1378,7 @@ int mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH - if (IS_AARCH64_RDM(cpuid_flags)) { + if (IS_AARCH64_RDM(mlkem_poly_cpuid_flags())) { /* Transform private key. All of result used in public key calculation. * Step 16: s_hat = NTT(s) */ for (i = 0; i < k; ++i) { @@ -1462,7 +1469,7 @@ int mlkem_encapsulate(const sword16* t, sword16* u, sword16* v, } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH - if (IS_AARCH64_RDM(cpuid_flags)) { + if (IS_AARCH64_RDM(mlkem_poly_cpuid_flags())) { /* Transform y. All of result used in calculation of u and v. * Step 18: y_hat <- NTT(y) */ for (i = 0; i < k; ++i) { @@ -1556,7 +1563,7 @@ int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, } #ifndef WOLFSSL_AARCH64_NO_SQRDMLSH - if (IS_AARCH64_RDM(cpuid_flags)) { + if (IS_AARCH64_RDM(mlkem_poly_cpuid_flags())) { /* Transform u. All of result used in calculation of w. * Step 6: ... NTT(u') */ for (i = 0; i < k; ++i) { @@ -1979,7 +1986,7 @@ int mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -1989,7 +1996,7 @@ int mlkem_keygen(sword16* s, sword16* t, sword16* e, const sword16* a, int k) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2200,7 +2207,7 @@ int mlkem_encapsulate(const sword16* pub, sword16* u, sword16* v, { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2209,7 +2216,7 @@ int mlkem_encapsulate(const sword16* pub, sword16* u, sword16* v, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2415,7 +2422,7 @@ int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2424,7 +2431,7 @@ int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -2498,20 +2505,20 @@ int mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, static WC_INLINE unsigned int mlkem_rej_uniform_n_ins(sword16* p, unsigned int len, const byte* r, unsigned int rLen) { - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI2 #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (IS_INTEL_AVX512_VBMI(cpuid_flags) && - IS_INTEL_AVX512_VBMI2(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI2(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_n_avx512_vbmi_vbmi2(p, len, r, rLen); } #endif - if (IS_INTEL_AVX512_VBMI2(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI2(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_n_avx512_vbmi2(p, len, r, rLen); } #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_n_avx512_vbmi(p, len, r, rLen); } #endif @@ -2522,20 +2529,20 @@ static WC_INLINE unsigned int mlkem_rej_uniform_n_ins(sword16* p, static WC_INLINE unsigned int mlkem_rej_uniform_ins(sword16* p, unsigned int len, const byte* r, unsigned int rLen) { - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI2 #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (IS_INTEL_AVX512_VBMI(cpuid_flags) && - IS_INTEL_AVX512_VBMI2(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI2(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_avx512_vbmi_vbmi2(p, len, r, rLen); } #endif - if (IS_INTEL_AVX512_VBMI2(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI2(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_avx512_vbmi2(p, len, r, rLen); } #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { return mlkem_rej_uniform_avx512_vbmi(p, len, r, rLen); } #endif @@ -2554,7 +2561,7 @@ static WC_INLINE unsigned int mlkem_rej_uniform_ins(sword16* p, static WC_INLINE void mlkem_redistribute_##N##_rand_ins(const word64* s, \ byte* r0, byte* r1, byte* r2, byte* r3) \ { \ - if (USE_INTEL_AVX512(cpuid_flags)) { \ + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { \ mlkem_redistribute_##N##_rand_avx512(s, r0, r1, r2, r3); \ return; \ } \ @@ -2575,7 +2582,7 @@ MLKEM_REDIST_INS(21) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 static WC_INLINE void mlkem_cbd_eta2_ins(sword16* p, const byte* r) { - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { mlkem_cbd_eta2_avx512(p, r); return; } @@ -2583,7 +2590,7 @@ static WC_INLINE void mlkem_cbd_eta2_ins(sword16* p, const byte* r) } static WC_INLINE void mlkem_cbd_eta3_ins(sword16* p, const byte* r) { - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { mlkem_cbd_eta3_avx512(p, r); return; } @@ -2921,10 +2928,10 @@ static int mlkem_gen_matrix_k3_avx2(sword16* a, byte* seed, int transposed) state[20] = W64LIT(0x8000000000000000); for (i = 0; i < GEN_MATRIX_SIZE; i += SHA3_128_BYTES) { #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); @@ -2944,10 +2951,10 @@ static int mlkem_gen_matrix_k3_avx2(sword16* a, byte* seed, int transposed) ctr0 = mlkem_rej_uniform_n_ins(a, MLKEM_N, rand, GEN_MATRIX_SIZE); while (ctr0 < MLKEM_N) { #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); @@ -3064,10 +3071,10 @@ static int mlkem_gen_matrix_k3_avx512(sword16* a, byte* seed, int transposed) state[20] = W64LIT(0x8000000000000000); for (i = 0; i < GEN_MATRIX_SIZE; i += SHA3_128_BYTES) { #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); @@ -3087,10 +3094,10 @@ static int mlkem_gen_matrix_k3_avx512(sword16* a, byte* seed, int transposed) ctr[0] = mlkem_rej_uniform_n_ins(a, MLKEM_N, rand, GEN_MATRIX_SIZE); while (ctr[0] < MLKEM_N) { #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); @@ -3788,10 +3795,10 @@ static int mlkem_prf(wc_Shake* shake256, byte* out, unsigned int outLen, /* Perform a block operation on the state for next block of output. */ #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { ForceZero(state, sizeof(state)); @@ -3860,10 +3867,10 @@ int mlkem_kdf(const byte* seed, int seedLen, byte* out, int outLen) state[WC_SHA3_256_COUNT - 1] = W64LIT(0x8000000000000000); #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { ForceZero(state, sizeof(state)); @@ -4320,7 +4327,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4329,7 +4336,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4357,7 +4364,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4366,7 +4373,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4394,7 +4401,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -4403,7 +4410,7 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5081,10 +5088,10 @@ static int mlkem_get_noise_eta2_avx2(MLKEM_PRF_T* prf, sword16* p, /* Perform a block operation on the state for next block of output. */ #ifndef WC_SHA3_NO_ASM - if (IS_INTEL_BMI2(cpuid_flags)) { + if (IS_INTEL_BMI2(mlkem_poly_cpuid_flags())) { sha3_block_bmi2(state); } - else if (IS_INTEL_AVX2(cpuid_flags)) { + else if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) { ForceZero(state, sizeof(state)); @@ -5905,7 +5912,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5914,7 +5921,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5947,7 +5954,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5956,7 +5963,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5985,7 +5992,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -5994,7 +6001,7 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6110,7 +6117,7 @@ int mlkem_cmp(const byte* a, const byte* b, int sz, int* fail) #else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6119,7 +6126,7 @@ int mlkem_cmp(const byte* a, const byte* b, int sz, int* fail) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6442,8 +6449,8 @@ int mlkem_vec_compress_10(byte* r, sword16* v, unsigned int k) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6453,7 +6460,7 @@ int mlkem_vec_compress_10(byte* r, sword16* v, unsigned int k) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6462,7 +6469,7 @@ int mlkem_vec_compress_10(byte* r, sword16* v, unsigned int k) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6583,7 +6590,7 @@ int mlkem_vec_compress_11(byte* r, sword16* v) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6592,7 +6599,7 @@ int mlkem_vec_compress_11(byte* r, sword16* v) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6703,8 +6710,8 @@ int mlkem_vec_decompress_10(sword16* v, const byte* b, unsigned int k) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6714,7 +6721,7 @@ int mlkem_vec_decompress_10(sword16* v, const byte* b, unsigned int k) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6723,7 +6730,7 @@ int mlkem_vec_decompress_10(sword16* v, const byte* b, unsigned int k) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6816,8 +6823,8 @@ int mlkem_vec_decompress_11(sword16* v, const byte* b) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6827,7 +6834,7 @@ int mlkem_vec_decompress_11(sword16* v, const byte* b) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -6836,7 +6843,7 @@ int mlkem_vec_decompress_11(sword16* v, const byte* b) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7013,8 +7020,8 @@ int mlkem_compress_4(byte* b, sword16* p) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7024,7 +7031,7 @@ int mlkem_compress_4(byte* b, sword16* p) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7033,7 +7040,7 @@ int mlkem_compress_4(byte* b, sword16* p) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7119,8 +7126,8 @@ int mlkem_compress_5(byte* b, sword16* p) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7130,7 +7137,7 @@ int mlkem_compress_5(byte* b, sword16* p) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7139,7 +7146,7 @@ int mlkem_compress_5(byte* b, sword16* p) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7214,7 +7221,7 @@ int mlkem_decompress_4(sword16* p, const byte* b) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7223,7 +7230,7 @@ int mlkem_decompress_4(sword16* p, const byte* b) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7307,7 +7314,7 @@ int mlkem_decompress_5(sword16* p, const byte* b) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7316,7 +7323,7 @@ int mlkem_decompress_5(sword16* p, const byte* b) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7398,7 +7405,7 @@ int mlkem_from_msg(sword16* p, const byte* msg) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7407,7 +7414,7 @@ int mlkem_from_msg(sword16* p, const byte* msg) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7513,7 +7520,7 @@ int mlkem_to_msg(byte* msg, sword16* p) { #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7523,7 +7530,7 @@ int mlkem_to_msg(byte* msg, sword16* p) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int svr_ret = SAVE_VECTOR_REGISTERS2(); if (svr_ret != 0) return svr_ret; @@ -7617,8 +7624,8 @@ int mlkem_from_bytes(sword16* p, const byte* b, int k) { #if defined(USE_INTEL_SPEEDUP) && !defined(WOLF_CRYPTO_CB_ONLY_MLKEM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -7635,7 +7642,7 @@ int mlkem_from_bytes(sword16* p, const byte* b, int k) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -7651,7 +7658,7 @@ int mlkem_from_bytes(sword16* p, const byte* b, int k) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -7721,8 +7728,8 @@ int mlkem_to_bytes(byte* b, sword16* p, int k) { #if defined(USE_INTEL_SPEEDUP) && !defined(WOLF_CRYPTO_CB_ONLY_MLKEM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512_VBMI - if (USE_INTEL_AVX512(cpuid_flags) && - IS_INTEL_AVX512_VBMI(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags()) && + IS_INTEL_AVX512_VBMI(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -7739,7 +7746,7 @@ int mlkem_to_bytes(byte* b, sword16* p, int k) else #endif #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 - if (USE_INTEL_AVX512(cpuid_flags)) { + if (USE_INTEL_AVX512(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2(); @@ -7755,7 +7762,7 @@ int mlkem_to_bytes(byte* b, sword16* p, int k) } else #endif - if (IS_INTEL_AVX2(cpuid_flags)) { + if (IS_INTEL_AVX2(mlkem_poly_cpuid_flags())) { int i; int svr_ret = SAVE_VECTOR_REGISTERS2();