Stop globally registering RSA key methods

.gitignore

@@ -82,6 +82,8 @@ tests/evp-sign
 tests/fork-change-slot
 tests/rsa-oaep
 tests/rsa-pss-sign
+tests/rsa-software-key
+tests/rsa-pss-direct
 tests/check-privkey
 tests/dup-key
 tests/check-privkey-prov

src/eng_back.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2002 Juha Yrjölä
  * Copyright (c) 2002 Olaf Kirch
  * Copyright (c) 2003 Kevin Stefanik
- * Copyright (c) 2016-2025 Michał Trojnara <Michal.Trojnara@stunnel.org>
+ * Copyright (c) 2016-2026 Michał Trojnara <Michal.Trojnara@stunnel.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -160,6 +160,28 @@ int ENGINE_CTX_finish(ENGINE_CTX *ctx)
 	return 1;
 }
 
+/* EVP_PKEY_set1_engine() is required for OpenSSL 1.1.x,
+ * but otherwise setting pkey->engine breaks OpenSSL 1.0.2 */
+#ifdef EVP_F_EVP_PKEY_SET1_ENGINE
+static int set_pkey_engine(PKCS11_KEY *key, EVP_PKEY *pkey, void *user_data)
+{
+	(void)key;
+	return EVP_PKEY_set1_engine(pkey, user_data) ? 0 : -1;
+}
+#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */
+
+int ENGINE_CTX_set_pkey_callback(ENGINE_CTX *ctx, ENGINE *engine)
+{
+#ifdef EVP_F_EVP_PKEY_SET1_ENGINE
+	return UTIL_CTX_set_pkey_callback(ctx->util_ctx,
+		PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY, set_pkey_engine, engine);
+#else
+	(void)ctx;
+	(void)engine;
+	return 1;
+#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */
+}
+
 /******************************************************************************/
 /* Engine load public/private key                                             */
 /******************************************************************************/

src/eng_front.c

@@ -5,7 +5,7 @@
  * Portions Copyright (c) 2003 Kevin Stefanik (kstef@mtppi.org)
  * Copied/modified by Kevin Stefanik (kstef@mtppi.org) for the OpenSC
  * project 2003.
- * Copyright (c) 2016-2025 Michał Trojnara <Michal.Trojnara@stunnel.org>
+ * Copyright (c) 2016-2026 Michał Trojnara <Michal.Trojnara@stunnel.org>
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -175,12 +175,13 @@ static EVP_PKEY *load_privkey(ENGINE *engine, const char *s_key_id,
 		UI_METHOD *ui_method, void *ui_data)
 {
 	ENGINE_CTX *ctx;
-	EVP_PKEY *pkey;
 
 	ctx = ENGINE_CTX_get(engine);
 	if (!ctx)
 		return 0;
-	bind_helper_methods(engine);
+	if (!bind_helper_methods(engine) ||
+			!ENGINE_CTX_set_pkey_callback(ctx, engine))
+		return 0;
 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
 	/*
 	 * A workaround for an OpenSSL bug affecting the handling of foreign
@@ -207,16 +208,7 @@ static EVP_PKEY *load_privkey(ENGINE *engine, const char *s_key_id,
 		}
 	}
 #endif
-	pkey = ENGINE_CTX_load_privkey(ctx, s_key_id, ui_method, ui_data);
-#ifdef EVP_F_EVP_PKEY_SET1_ENGINE
-	/* EVP_PKEY_set1_engine() is required for OpenSSL 1.1.x,
-	 * but otherwise setting pkey->engine breaks OpenSSL 1.0.2 */
-	if (pkey && !EVP_PKEY_set1_engine(pkey, engine)) {
-		EVP_PKEY_free(pkey);
-		pkey = NULL;
-	}
-#endif /* EVP_F_EVP_PKEY_SET1_ENGINE */
-	return pkey;
+	return ENGINE_CTX_load_privkey(ctx, s_key_id, ui_method, ui_data);
 }
 
 static int engine_ctrl(ENGINE *engine, int cmd, long i, void *p, void (*f) (void))

src/engine.h

@@ -2,7 +2,7 @@
  * Copyright (c) 2001 Markus Friedl
  * Copyright (c) 2002 Juha Yrjölä
  * Copyright (c) 2003 Kevin Stefanik
- * Copyright (c) 2016-2025 Michał Trojnara
+ * Copyright (c) 2016-2026 Michał Trojnara
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -84,6 +84,8 @@ int ENGINE_CTX_init(ENGINE_CTX *ctx);
 
 int ENGINE_CTX_finish(ENGINE_CTX *ctx);
 
+int ENGINE_CTX_set_pkey_callback(ENGINE_CTX *ctx, ENGINE *engine);
+
 int ENGINE_CTX_ctrl(ENGINE_CTX *ctx, int cmd, long i, void *p, void (*f)(void));
 
 EVP_PKEY *ENGINE_CTX_load_pubkey(ENGINE_CTX *ctx, const char *s_key_id,

src/libp11-int.h

@@ -1,7 +1,7 @@
 /* libp11, a simple layer on top of PKCS#11 API
  * Copyright (C) 2005 Olaf Kirch <okir@lst.de>
  * Copyright (C) 2015-2025 Michał Trojnara <Michal.Trojnara@stunnel.org>
- * Copyright © 2025 Mobi - Com Polska Sp. z o.o.
+ * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o.
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
@@ -63,6 +63,8 @@ typedef struct pkcs11_keys PKCS11_keys;
 typedef struct pkcs11_object_ops PKCS11_OBJECT_ops;
 typedef struct pkcs11_template_st PKCS11_TEMPLATE;
 
+#define PKCS11_PKEY_CALLBACK_COUNT 2
+
 /* get private implementations of PKCS11 structures */
 
 /*
@@ -81,6 +83,8 @@ struct pkcs11_ctx_private {
 	unsigned int forkid;
 	int initialized;
 	void (*vlog_a)(int, const char *, va_list); /* for the logging callback */
+	PKCS11_PKEY_CALLBACK pkey_callbacks[PKCS11_PKEY_CALLBACK_COUNT];
+	void *pkey_callback_data[PKCS11_PKEY_CALLBACK_COUNT];
 };
 
 struct pkcs11_keys {
@@ -626,11 +630,6 @@ extern void pkcs11_ed_key_method_free(void);
 extern void pkcs11_xdh_key_method_free(void);
 #endif /* !defined(OPENSSL_NO_ECX) && OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L
-/* Free the global RSA EVP_PKEY_METHOD */
-extern void pkcs11_rsa_key_method_free(void);
-# endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */
-
 #if OPENSSL_VERSION_NUMBER < 0x100020d0L || defined(LIBRESSL_VERSION_NUMBER)
 /* Get sign_init and sign callbacks from EVP_PKEY_METHOD */
 extern void EVP_PKEY_meth_get_sign(EVP_PKEY_METHOD *pmeth,

src/libp11.exports

@@ -1,6 +1,7 @@
 PKCS11_CTX_init_args
 PKCS11_CTX_new_ex
 PKCS11_CTX_new
+PKCS11_CTX_set_pkey_callback
 PKCS11_CTX_load
 PKCS11_CTX_unload
 PKCS11_CTX_free

src/libp11.h

@@ -1,6 +1,6 @@
 /* libp11, a simple layer on top of PKCS#11 API
  * Copyright (C) 2005 Olaf Kirch <okir@lst.de>
- * Copyright © 2025 Mobi - Com Polska Sp. z o.o.
+ * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o.
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
@@ -171,6 +171,18 @@ struct PKCS11_kgen_attrs_st {
 /** PKCS11 ASCII logging callback */
 typedef void (*PKCS11_VLOG_A_CB)(int, const char *, va_list);
 
+/**
+ * Callback invoked for an EVP_PKEY returned by libp11
+ *
+ * The key arguments are borrowed and must not be freed by the callback.
+ * The callback may modify the EVP_PKEY and must return 0 on success or -1
+ * on error.
+ */
+typedef int (*PKCS11_PKEY_CALLBACK)(PKCS11_KEY *, EVP_PKEY *, void *);
+
+/** Callback type for PKCS11_get_private_key() */
+#define PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY 1
+
 /**
  * Create a new libp11 context with specified flags
  *
@@ -187,6 +199,23 @@ extern PKCS11_CTX *PKCS11_CTX_new_ex(int flags);
  */
 extern PKCS11_CTX *PKCS11_CTX_new(void);
 
+/**
+ * Set a callback for EVP_PKEY objects returned by this context
+ *
+ * The callback and its user data must remain valid until they are replaced,
+ * unset, or the context is freed. Callback registration must not be changed
+ * concurrently with key retrieval.
+ *
+ * @param ctx context allocated by PKCS11_CTX_new()
+ * @param callback_type one of PKCS11_PKEY_CALLBACK_* types
+ * @param callback callback function, or NULL to unset it
+ * @param user_data opaque callback data
+ * @retval 0 success
+ * @retval -1 unsupported callback type or invalid context
+ */
+extern int PKCS11_CTX_set_pkey_callback(PKCS11_CTX *ctx,
+	int callback_type, PKCS11_PKEY_CALLBACK callback, void *user_data);
+
 /**
  * Specify any private PKCS#11 module initialization args, if necessary
  *

src/p11_front.c

@@ -1,6 +1,6 @@
 /* libp11, a simple layer on top of PKCS#11 API
  * Copyright (C) 2016-2025 Michał Trojnara <Michal.Trojnara@stunnel.org>
- * Copyright © 2025 Mobi - Com Polska Sp. z o.o.
+ * Copyright © 2025-2026 Mobi - Com Polska Sp. z o.o.
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
@@ -42,6 +42,22 @@ PKCS11_CTX *PKCS11_CTX_new(void)
 	return pkcs11_CTX_new(0);
 }
 
+int PKCS11_CTX_set_pkey_callback(PKCS11_CTX *pctx,
+		int callback_type, PKCS11_PKEY_CALLBACK callback, void *user_data)
+{
+	PKCS11_CTX_private *ctx;
+
+	if (!pctx || callback_type <= 0 ||
+			callback_type >= PKCS11_PKEY_CALLBACK_COUNT)
+		return -1;
+	ctx = pctx->_private;
+	if (check_fork(ctx) < 0)
+		return -1;
+	ctx->pkey_callbacks[callback_type] = callback;
+	ctx->pkey_callback_data[callback_type] = callback ? user_data : NULL;
+	return 0;
+}
+
 void PKCS11_CTX_init_args(PKCS11_CTX *ctx, const char *init_args)
 {
 	if (check_fork(ctx->_private) < 0)
@@ -255,9 +271,22 @@ int PKCS11_get_key_type(PKCS11_KEY *pkey)
 EVP_PKEY *PKCS11_get_private_key(PKCS11_KEY *pkey)
 {
 	PKCS11_OBJECT_private *key = pkey->_private;
+	PKCS11_CTX_private *ctx = key->slot->ctx;
+	PKCS11_PKEY_CALLBACK callback;
+	EVP_PKEY *ret;
+
 	if (check_object_fork(key) < 0)
 		return NULL;
-	return pkcs11_get_key(key, CKO_PRIVATE_KEY);
+	ret = pkcs11_get_key(key, CKO_PRIVATE_KEY);
+	if (!ret)
+		return NULL;
+	callback = ctx->pkey_callbacks[PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY];
+	if (callback && callback(pkey, ret,
+			ctx->pkey_callback_data[PKCS11_PKEY_CALLBACK_GET_PRIVATE_KEY])) {
+		EVP_PKEY_free(ret);
+		return NULL;
+	}
+	return ret;
 }
 
 EVP_PKEY *PKCS11_get_public_key(PKCS11_KEY *pkey)

src/p11_load.c

@@ -39,9 +39,6 @@ static void libp11_global_free(void)
 
 #ifndef OPENSSL_NO_RSA
 	pkcs11_rsa_method_free();
-# if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L
-	pkcs11_rsa_key_method_free();
-# endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */
 #endif /* OPENSSL_NO_RSA */
 
 #if !defined(OPENSSL_NO_ECX) && OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L

src/p11_rsa.c

@@ -29,10 +29,6 @@
 static int rsa_ex_index = 0;
 static RSA_METHOD *pkcs11_rsa_method = NULL;
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L
-static EVP_PKEY_METHOD *pkey_method_rsa  = NULL;
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */
-
 static RSA *pkcs11_get1_rsa(PKCS11_OBJECT_private *key)
 {
 	EVP_PKEY *evp_key = pkcs11_get_key(key, key->object_class);
@@ -235,39 +231,6 @@ void pkcs11_set_ex_data_rsa(RSA *rsa, PKCS11_OBJECT_private *key)
 	RSA_set_ex_data(rsa, rsa_ex_index, key);
 }
 
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L
-
-/* Global initialize RSA EVP_PKEY_METHOD */
-static int pkcs11_pkey_method_rsa_new(void)
-{
-	if (pkey_method_rsa)
-		return 1; /* EVP_PKEY_RSA method already initialized */
-
-	pkey_method_rsa = pkcs11_pkey_method_rsa();
-	if (!pkey_method_rsa)
-		return 0;
-
-	/* Register the method globally */
-	if (!EVP_PKEY_meth_add0(pkey_method_rsa)) {
-		EVP_PKEY_meth_free(pkey_method_rsa);
-		pkey_method_rsa = NULL;
-		return 0;
-	}
-	return 1;
-}
-
-void pkcs11_rsa_key_method_free(void)
-{
-	if (pkey_method_rsa) {
-		free_pkey_ex_index();
-		EVP_PKEY_meth_remove(pkey_method_rsa);
-		EVP_PKEY_meth_free(pkey_method_rsa);
-		pkey_method_rsa = NULL;
-	}
-}
-
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x40000000L */
-
 /*
  * Build an EVP_PKEY object
  */
@@ -285,21 +248,6 @@ static EVP_PKEY *pkcs11_get_evp_key_rsa(PKCS11_OBJECT_private *key)
 		return NULL;
 	}
 	if (key->object_class == CKO_PRIVATE_KEY) {
-#if OPENSSL_VERSION_NUMBER >= 0x30000000L
-# if OPENSSL_VERSION_NUMBER < 0x40000000L
-		if ((key->slot->ctx->flags & PKCS11_FLAG_NO_METHODS) == 0) {
-			/* global initialize RSA EVP_PKEY_METHOD */
-			if (!pkcs11_pkey_method_rsa_new()) {
-				EVP_PKEY_free(pk);
-				return NULL;
-			}
-			alloc_pkey_ex_index();
-			pkcs11_set_ex_data_pkey(pk, key);
-			atexit(pkcs11_rsa_key_method_free);
-		}
-# endif /* OPENSSL_VERSION_NUMBER < 0x40000000L */
-#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
-
 		/* The RSA object owns the reference stored in its ex_data. */
 		key = pkcs11_object_ref(key);
 		RSA_set_method(rsa, PKCS11_get_rsa_method());
@@ -318,7 +266,11 @@ static EVP_PKEY *pkcs11_get_evp_key_rsa(PKCS11_OBJECT_private *key)
 #endif
 	pkcs11_set_ex_data_rsa(rsa, key);
 
-	EVP_PKEY_set1_RSA(pk, rsa); /* Also increments the rsa ref count */
+	if (!EVP_PKEY_set1_RSA(pk, rsa)) {
+		RSA_free(rsa);
+		EVP_PKEY_free(pk);
+		return NULL;
+	}
 	RSA_free(rsa); /* Drops our reference to it */
 	return pk;
 }

src/util.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2025 Michał Trojnara <Michal.Trojnara@stunnel.org>
+ * Copyright (c) 2025-2026 Michał Trojnara <Michal.Trojnara@stunnel.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -58,6 +58,8 @@ void UTIL_CTX_free(UTIL_CTX *ctx);
 int UTIL_CTX_set_module(UTIL_CTX *ctx, const char *module);
 int UTIL_CTX_set_init_args(UTIL_CTX *ctx, const char *init_args);
 int UTIL_CTX_set_ui_method(UTIL_CTX *ctx, UI_METHOD *ui_method, void *ui_data);
+int UTIL_CTX_set_pkey_callback(UTIL_CTX *ctx, int callback_type,
+	PKCS11_PKEY_CALLBACK callback, void *user_data);
 int UTIL_CTX_enumerate_slots(UTIL_CTX *ctx);
 void UTIL_CTX_free_libp11(UTIL_CTX *ctx);