ssl: conditionally enable SSL Engine APIs

Conditionally enable ssl_socket methods that use the deprecated OpenSSL
ENGINE APIs. The APIs are enabled when the OpenSSL being compiled
against has the ENGINE APIs enabled. In addition these APIs can be
disabled by passing -DENABLE_SSL_ENGINE_API=OFF to CMake at build-time.

Fixed: #795
Fixed: #713

Signed-off-by: GitHub <noreply@github.com>

CMakeLists.txt

@@ -42,6 +42,7 @@ endif()
 
 include(CheckSymbolExists)
 include(CheckLibraryExists)
+include(CMakeDependentOption)
 include(CMakePushCheckState)
 include(GNUInstallDirs)
 
@@ -120,14 +121,19 @@ if (ENABLE_SSL_SUPPORT)
   set(THREADS_PREFER_PTHREAD_FLAG ON)
   find_package(Threads REQUIRED)
   cmake_pop_check_state()
+
+  cmake_push_check_state()
+  set(CMAKE_REQUIRED_LIBRARIES OpenSSL::SSL)
+  check_symbol_exists(ENGINE_new openssl/engine.h HAS_OPENSSL_ENGINE)
+  cmake_pop_check_state()
+
+  cmake_dependent_option(ENABLE_SSL_ENGINE_API "Enable support for deprecated OpenSSL ENGINE feature" ON "HAS_OPENSSL_ENGINE" OFF)
 endif()
 
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   include(CTest)
 endif()
 
-include(CMakeDependentOption)
-
 option(BUILD_SHARED_LIBS "Build rabbitmq-c as a shared library" ON)
 option(BUILD_STATIC_LIBS "Build rabbitmq-c as a static library" ON)
 option(INSTALL_STATIC_LIBS "Install rabbitmq-c static library" ON)

include/rabbitmq-c/config.h.in

@@ -0,0 +1,9 @@
+// Copyright 2023, Alan Antonuk and the rabbitmq-c contributors.
+// SPDX-License-Identifier: mit
+
+#ifndef RABBITMQ_C_CONFIG_H
+#define RABBITMQ_C_CONFIG_H
+
+#cmakedefine AMQP_SSL_ENGINE_API_ENABLED
+
+#endif  /* RABBITMQ_C_CONFIG_H */

include/rabbitmq-c/ssl_socket.h

@@ -7,6 +7,7 @@
 #define RABBITMQ_C_SSL_SOCKET_H
 
 #include <rabbitmq-c/amqp.h>
+#include <rabbitmq-c/config.h>
 #include <rabbitmq-c/export.h>
 
 AMQP_BEGIN_DECLS
@@ -105,6 +106,7 @@ AMQP_EXPORT
 int AMQP_CALL amqp_ssl_socket_set_key(amqp_socket_t *self, const char *cert,
                                       const char *key);
 
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 /**
  * Set the client key use the engine.
  *
@@ -122,6 +124,7 @@ int AMQP_CALL amqp_ssl_socket_set_key(amqp_socket_t *self, const char *cert,
 AMQP_EXPORT
 int AMQP_CALL amqp_ssl_socket_set_key_engine(amqp_socket_t *self,
                                              const char *cert, const char *key);
+#endif
 
 /**
  * Set the client key from a buffer.
@@ -269,6 +272,7 @@ void AMQP_CALL amqp_set_initialize_ssl_library(amqp_boolean_t do_initialize);
 AMQP_DEPRECATED_EXPORT
 int AMQP_CALL amqp_initialize_ssl_library(void);
 
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 /**
  * Set the engine for underlying SSL/TLS library.
  *
@@ -284,6 +288,7 @@ int AMQP_CALL amqp_initialize_ssl_library(void);
  */
 AMQP_EXPORT
 int amqp_set_ssl_engine(const char *engine);
+#endif
 
 /**
  * Uninitialize the underlying SSL/TLS library.

librabbitmq/CMakeLists.txt

@@ -24,6 +24,9 @@ if (ENABLE_SSL_SUPPORT)
     set(AMQP_SSL_SRCS ${AMQP_SSL_SRCS} unix/threads.h)
     set(SSL_INCLUDE_DIRS unix)
   endif()
+  if (ENABLE_SSL_ENGINE_API)
+    set(AMQP_SSL_ENGINE_API_ENABLED ON)
+  endif()
 endif()
 
 set(PUBLIC_INCLUDE_DIRS
@@ -167,6 +170,11 @@ if(BUILD_STATIC_LIBS)
   add_library(rabbitmq::rabbitmq-static ALIAS rabbitmq-static)
 endif()
 
+configure_file(
+  ../include/rabbitmq-c/config.h.in
+  ${CMAKE_CURRENT_BINARY_DIR}/../include/rabbitmq-c/config.h
+)
+
 include(GenerateExportHeader)
 generate_export_header(${RMQ_GEN_EXPORT_TARGET}
   BASE_NAME AMQP
@@ -189,6 +197,7 @@ install(FILES
   ../include/rabbitmq-c/framing.h
   ../include/rabbitmq-c/tcp_socket.h
   ${AMQP_SSL_SOCKET_H_PATH}
+  ${CMAKE_CURRENT_BINARY_DIR}/../include/rabbitmq-c/config.h
   ${CMAKE_CURRENT_BINARY_DIR}/../include/rabbitmq-c/export.h
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/rabbitmq-c
   COMPONENT rabbitmq-c-development

librabbitmq/amqp_openssl.c

@@ -16,14 +16,17 @@
 #include "amqp_private.h"
 #include "amqp_socket.h"
 #include "amqp_time.h"
+#include "rabbitmq-c/config.h"
 #include "rabbitmq-c/ssl_socket.h"
 #include "threads.h"
 
 #include <ctype.h>
 #include <limits.h>
 #include <openssl/bio.h>
 #include <openssl/conf.h>
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 #include <openssl/engine.h>
+#endif
 #include <openssl/err.h>
 #include <openssl/rsa.h>
 #include <openssl/ssl.h>
@@ -37,7 +40,9 @@ static int decrement_ssl_connections(void);
 static pthread_mutex_t openssl_init_mutex = PTHREAD_MUTEX_INITIALIZER;
 static amqp_boolean_t openssl_bio_initialized = 0;
 static int openssl_connections = 0;
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 static ENGINE *openssl_engine = NULL;
+#endif
 
 #define CHECK_SUCCESS(condition)                                            \
   do {                                                                      \
@@ -405,6 +410,7 @@ int amqp_ssl_socket_set_key(amqp_socket_t *base, const char *cert,
   return AMQP_STATUS_OK;
 }
 
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 int amqp_ssl_socket_set_key_engine(amqp_socket_t *base, const char *cert,
                                    const char *key) {
   int status;
@@ -432,6 +438,7 @@ int amqp_ssl_socket_set_key_engine(amqp_socket_t *base, const char *cert,
   }
   return AMQP_STATUS_OK;
 }
+#endif
 
 static int password_cb(AMQP_UNUSED char *buffer, AMQP_UNUSED int length,
                        AMQP_UNUSED int rwflag, AMQP_UNUSED void *user_data) {
@@ -583,6 +590,7 @@ void amqp_set_initialize_ssl_library(amqp_boolean_t do_initialize) {
 
 int amqp_initialize_ssl_library(void) { return AMQP_STATUS_OK; }
 
+#ifdef AMQP_SSL_ENGINE_API_ENABLED
 int amqp_set_ssl_engine(const char *engine) {
   int status = AMQP_STATUS_OK;
   CHECK_SUCCESS(pthread_mutex_lock(&openssl_init_mutex));
@@ -614,6 +622,7 @@ int amqp_set_ssl_engine(const char *engine) {
   CHECK_SUCCESS(pthread_mutex_unlock(&openssl_init_mutex));
   return status;
 }
+#endif
 
 static int initialize_ssl_and_increment_connections() {
   int status;