fix: raise wolfSSH RSA user-authentication key minimum

Raise a cryptographic minimum to current guidance. It rejects peers below
the new floor by design.

- Reject RSA user-authentication keys smaller than 2048 bits with
  WS_CERT_KEY_SIZE_E, per NIST SP 800-131A. The floor is configurable via
  a new WOLFSSH_RSA_MIN_KEY_BITS define (default 2048).

Issue: F-6517

Author: MarkAtwood

src/internal.c

@@ -8458,6 +8458,28 @@ static int DoUserAuthRequestPassword(WOLFSSH* ssh, WS_UserAuthData* authData,
 }
 
 #ifndef WOLFSSH_NO_RSA
+/* Utility for the RSA user auth paths. */
+/* returns WS_SUCCESS when the key clears WOLFSSH_RSA_MIN_KEY_BITS. */
+static int CheckRsaKeyBits(RsaKey* key)
+{
+    int ret = WS_SUCCESS;
+    int keyBits;
+
+    /* The encrypt size rounds up to a byte; offload builds leave n empty. */
+    keyBits = mp_count_bits(&key->n);
+    if (keyBits == 0) {
+        keyBits = wc_RsaEncryptSize(key) * 8;
+    }
+
+    if (keyBits < WOLFSSH_RSA_MIN_KEY_BITS) {
+        WLOG(WS_LOG_DEBUG, "RSA auth key too small (%d bits)", keyBits);
+        ret = WS_CERT_KEY_SIZE_E;
+    }
+
+    return ret;
+}
+
+
 /* Utility for DoUserAuthRequestPublicKey() */
 /* returns negative for error, positive is size of digest. */
 static int DoUserAuthRequestRsa(WOLFSSH* ssh, WS_UserAuthData_PublicKey* pk,
@@ -8546,6 +8568,10 @@ static int DoUserAuthRequestRsa(WOLFSSH* ssh, WS_UserAuthData_PublicKey* pk,
         }
     }
 
+    if (ret == WS_SUCCESS) {
+        ret = CheckRsaKeyBits(key);
+    }
+
     if (ret == WS_SUCCESS) {
         i = 0;
         /* Check that the signature's pubkey type matches the expected one. */
@@ -8705,11 +8731,7 @@ static int DoUserAuthRequestRsaCert(WOLFSSH* ssh, WS_UserAuthData_PublicKey* pk,
     }
 
     if (ret == WS_SUCCESS) {
-        int keySz = wc_RsaEncryptSize(key) * 8;
-        if (keySz < 2048) {
-            WLOG(WS_LOG_DEBUG, "Key size too small (%d)", keySz);
-            ret = WS_CERT_KEY_SIZE_E;
-        }
+        ret = CheckRsaKeyBits(key);
     }
 
     if (ret == WS_SUCCESS) {

wolfssh/internal.h

@@ -566,6 +566,13 @@ enum NameIdType {
 #ifndef WOLFSSH_DEFAULT_GEXDH_MIN
     #define WOLFSSH_DEFAULT_GEXDH_MIN 2048
 #endif
+
+#ifndef WOLFSSH_RSA_MIN_KEY_BITS
+    /* Minimum accepted RSA public-key size (bits) for user authentication.
+     * Per NIST SP 800-131A; override at build time if a smaller key must be
+     * accepted. */
+    #define WOLFSSH_RSA_MIN_KEY_BITS 2048
+#endif
 #ifndef WOLFSSH_DEFAULT_GEXDH_PREFERRED
     #define WOLFSSH_DEFAULT_GEXDH_PREFERRED 3072
 #endif