Fix OpenSSL builds with deprecated APIs disabled

.github/workflows/openssl_compat.yml

@@ -0,0 +1,32 @@
+name: openssl_compat
+on:
+  push:
+    paths-ignore:
+    - 'docs/**'
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  openssl_compat:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+        - image: debian:bullseye-slim
+          api: 0x10100000L
+        - image: debian:bookworm-slim
+          api: 0x30000000L
+    container: ${{ matrix.image }}
+    steps:
+    - name: use archived repositories for OpenSSL 1.1.1
+      if: matrix.api == '0x10100000L'
+      run: |
+        printf 'deb http://archive.debian.org/debian bullseye main\ndeb http://archive.debian.org/debian-security bullseye-security main\n' > /etc/apt/sources.list
+        rm -f /etc/apt/sources.list.d/debian.sources
+        printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99archive
+    - name: install dependencies
+      run: apt-get update && apt-get install -y --no-install-recommends g++ make cmake ninja-build git ca-certificates python3 openssl libssl-dev zlib1g-dev
+    - uses: actions/checkout@v4
+    - name: make test
+      run: make -f makefile.dev test_openssl_no_deprecated OPENSSL_API_COMPAT=${{ matrix.api }}

docs/build.md

@@ -87,6 +87,16 @@ It is also possible to externally include the project, so that everything is fet
     )
 ```
 
+### OpenSSL compatibility checks
+
+`make -f makefile.dev test_openssl_no_deprecated` builds the library, CLI, and
+tests against OpenSSL 3 with `OPENSSL_NO_DEPRECATED` and a 3.0 API compatibility
+level. CI runs the same target against OpenSSL 1.1.1 and 3.0 using Debian
+Bullseye and Bookworm; the 1.1.1 build uses `OPENSSL_API_COMPAT=0x10100000L`.
+This checks that modern builds do not depend on compatibility declarations
+for older initialization, locking, or ECDH APIs. OpenSSL 3 uses the owned-reference
+certificate getter; older OpenSSL and LibreSSL retain their existing API.
+
 ### vcpkg
 
 It is possible to get IXWebSocket through Microsoft [vcpkg](https://github.com/microsoft/vcpkg).

ixwebsocket/IXSocketOpenSSL.cpp

@@ -94,7 +94,9 @@ namespace ix
 
     std::atomic<bool> SocketOpenSSL::_openSSLInitializationSuccessful(false);
     std::once_flag SocketOpenSSL::_openSSLInitFlag;
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
     std::vector<std::unique_ptr<std::mutex>> openSSLMutexes;
+#endif
 
     SocketOpenSSL::SocketOpenSSL(const SocketTLSOptions& tlsOptions, int fd)
         : Socket(fd)
@@ -126,14 +128,15 @@ namespace ix
             }
             CRYPTO_set_locking_callback(SocketOpenSSL::openSSLLockingCallback);
         }
-#endif
 
         (void) OpenSSL_add_ssl_algorithms();
         (void) SSL_load_error_strings();
+#endif
 
         _openSSLInitializationSuccessful = true;
     }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
     void SocketOpenSSL::openSSLLockingCallback(int mode,
                                                int type,
                                                const char* /*file*/,
@@ -148,6 +151,7 @@ namespace ix
             openSSLMutexes[type]->unlock();
         }
     }
+#endif
 
     std::string SocketOpenSSL::getSSLError(int ret)
     {
@@ -313,7 +317,11 @@ namespace ix
 #endif
                                                std::string& errMsg)
     {
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L && !defined(LIBRESSL_VERSION_NUMBER)
+        X509* server_cert = SSL_get1_peer_certificate(ssl);
+#else
         X509* server_cert = SSL_get_peer_certificate(ssl);
+#endif
         if (server_cert == nullptr)
         {
             errMsg = "OpenSSL failed - peer didn't present a X509 certificate.";
@@ -719,7 +727,9 @@ namespace ix
                 return false;
             }
 
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
             SSL_set_ecdh_auto(_ssl_connection, 1);
+#endif
 
             SSL_set_fd(_ssl_connection, _sockfd);
 

ixwebsocket/IXSocketOpenSSL.h

@@ -53,8 +53,9 @@ namespace ix
         bool openSSLServerHandshake(std::string& errMsg,
                                     const CancellationRequest& isCancellationRequested);
 
-        // Required for OpenSSL < 1.1
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
         static void openSSLLockingCallback(int mode, int type, const char* /*file*/, int /*line*/);
+#endif
 
         SSL* _ssl_connection;
         SSL_CTX* _ssl_context;

makefile.dev

@@ -260,3 +260,11 @@ AUTOBAHN_ARGS ?=
 .PHONY: test_autobahn_interactive
 test_autobahn_interactive:
 	python3 tools/run_autobahn.py --interactive $(AUTOBAHN_ARGS)
+
+# Hide compatibility APIs so OpenSSL migrations fail in CI rather than downstream builds.
+OPENSSL_API_COMPAT ?= 0x30000000L
+.PHONY: test_openssl_no_deprecated
+test_openssl_no_deprecated:
+	cmake -S . -B build -GNinja -UOPENSSL_CRYPTO_LIBRARY -UOPENSSL_SSL_LIBRARY -UOPENSSL_INCLUDE_DIR -DCMAKE_BUILD_TYPE=Debug -DCMAKE_UNITY_BUILD=ON -DUSE_TLS=ON -DUSE_OPEN_SSL=ON -DUSE_LIBRE_SSL=OFF -DUSE_MBED_TLS=OFF -DUSE_NETWORK_FRAMEWORK=OFF -DUSE_SECURE_TRANSPORT=OFF -DUSE_ZLIB=ON -DUSE_TEST=ON -DUSE_WS=ON -DCMAKE_CXX_STANDARD_REQUIRED=ON -DCMAKE_CXX_FLAGS="-DOPENSSL_NO_DEPRECATED -DOPENSSL_API_COMPAT=$(OPENSSL_API_COMPAT) -Werror=deprecated-declarations"
+	cmake --build build --parallel 2
+	ctest --test-dir build --output-on-failure

test/generate_network_certs.sh

@@ -4,6 +4,10 @@ set -euo pipefail
 openssl_bin=$1
 cert_dir=$2
 mkdir -p "$cert_dir"
+# OpenSSL 1.1.1 appends -addext to system defaults, which can duplicate CA
+# extensions and produce invalid certificates. Use a test-only configuration.
+printf '[req]\ndistinguished_name=dn\n[dn]\n' > "$cert_dir/openssl.cnf"
+export OPENSSL_CONF="$cert_dir/openssl.cnf"
 "$openssl_bin" req -x509 -newkey rsa:2048 -nodes -sha256 -days 3650 \
     -subj /CN=IXWebSocket-Test-CA -addext basicConstraints=critical,CA:TRUE -addext keyUsage=critical,keyCertSign,cRLSign -keyout "$cert_dir/ca.key" -out "$cert_dir/ca.pem" >/dev/null 2>&1
 for name in server wrong client; do