Workaround: Decode with non-zero selections for affected OpenSSL vers…

…ions.

This is a workaround for OpenSSL versions affected by the decoding issue. Avoid
decoding with selection 0 for the OpenSSL versions to avoid the decoding issue.

ext/openssl/ossl_pkey.c

@@ -83,23 +83,83 @@ ossl_pkey_new(EVP_PKEY *pkey)
 # include <openssl/decoder.h>
 
 EVP_PKEY *
-ossl_pkey_read_generic(BIO *bio, VALUE pass)
+ossl_pkey_read(BIO *bio, const char *input_type, int selection, VALUE pass)
 {
-    void *ppass = (void *)pass;
-    OSSL_DECODER_CTX *dctx;
     EVP_PKEY *pkey = NULL;
-    int pos = 0, pos2;
+    OSSL_DECODER_CTX *dctx = NULL;
+    void *ppass = (void *)pass;
+    int pos = 0, pos2 = 0;
 
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "DER", NULL, NULL, 0, NULL, NULL);
+    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, input_type, NULL, NULL, selection, NULL, NULL);
     if (!dctx)
         goto out;
     if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
         goto out;
+    while (1) {
+        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+            goto out;
+        if (BIO_eof(bio))
+            break;
+        pos2 = BIO_tell(bio);
+        if (pos2 < 0 || pos2 <= pos)
+            break;
+        ossl_clear_error();
+        pos = pos2;
+    }
+  out:
+    OSSL_BIO_reset(bio);
+    OSSL_DECODER_CTX_free(dctx);
+    return pkey;
+}
+
+EVP_PKEY *
+ossl_pkey_read_generic(BIO *bio, VALUE pass)
+{
+    EVP_PKEY *pkey = NULL;
+    /* Selections to try. */
+    int *selections = NULL;
+    int i = 0;
+    int selection_num = 0;
+
+    /*
+     * A workaround for the decoder failing to decode or returning bogus keys
+     * with selection 0, if a key management provider is different from a
+     * decoder provider. The workaround is to avoid using selection 0.
+     * If the number of providers is more than equal 2, apply the workaround
+     * simply.
+     *
+     * Affected OpenSSL versions: >= 3.1.0, <= 3.1.2, or >= 3.0.0, <= 3.0.10
+     * Fixed OpenSSL versions: 3.2, next release of the 3.1.z and 3.0.z
+     *
+     * See <https://github.com/openssl/openssl/pull/21519> for details.
+     */
+    if ((
+        (OSSL_OPENSSL_PREREQ(3, 1, 0, 0) && !OSSL_OPENSSL_PREREQ(3, 1, 0, 3)) ||
+        (OSSL_OPENSSL_PREREQ(3, 0, 0, 0) && !OSSL_OPENSSL_PREREQ(3, 0, 0, 11))
+        ) && ossl_count_provider_number() >= 2) {
+        /* See EVP_PKEY_fromdata(3) - Selections to see all the selections. */
+        selection_num = 3;
+        if (!(selections = (int *)calloc(selection_num, sizeof(int)))) {
+            ossl_raise(ePKeyError, "Failed to allocate selections");
+        }
+        selections[0] = EVP_PKEY_KEY_PARAMETERS;
+        selections[1] = EVP_PKEY_PUBLIC_KEY;
+        selections[2] = EVP_PKEY_KEYPAIR;
+    } else {
+        selection_num = 1;
+        if (!(selections = (int *)calloc(selection_num, sizeof(int)))) {
+            ossl_raise(ePKeyError, "Failed to allocate selections");
+        }
+        selections[0] = 0;
+    }
 
     /* First check DER */
-    if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-        goto out;
-    OSSL_BIO_reset(bio);
+    for (i = 0; i < selection_num; i++) {
+        pkey = ossl_pkey_read(bio, "DER", selections[i], pass);
+        if (pkey) {
+            goto out;
+        }
+    }
 
     /*
      * Then check PEM; multiple OSSL_DECODER_from_bio() calls may be needed.
@@ -128,48 +188,21 @@ ossl_pkey_read_generic(BIO *bio, VALUE pass)
      * we use the different selection as a workaround.
      * https://github.com/openssl/openssl/issues/20657
      */
-    OSSL_DECODER_CTX_free(dctx);
-    dctx = NULL;
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL,
-                                         EVP_PKEY_KEYPAIR, NULL, NULL);
-    if (!dctx)
+    pkey = ossl_pkey_read(bio, "PEM", EVP_PKEY_KEYPAIR, pass);
+    if (pkey) {
         goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
-        goto out;
-    while (1) {
-        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
-            goto out;
-        if (BIO_eof(bio))
-            break;
-        pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
-            break;
-        ossl_clear_error();
-        pos = pos2;
     }
 
-    OSSL_BIO_reset(bio);
-    OSSL_DECODER_CTX_free(dctx);
-    dctx = NULL;
-    dctx = OSSL_DECODER_CTX_new_for_pkey(&pkey, "PEM", NULL, NULL, 0, NULL, NULL);
-    if (!dctx)
-        goto out;
-    if (OSSL_DECODER_CTX_set_pem_password_cb(dctx, ossl_pem_passwd_cb, ppass) != 1)
-        goto out;
-    while (1) {
-        if (OSSL_DECODER_from_bio(dctx, bio) == 1)
+    for (i = 0; i < selection_num; i++) {
+        pkey = ossl_pkey_read(bio, "PEM", selections[i], pass);
+        if (pkey) {
             goto out;
-        if (BIO_eof(bio))
-            break;
-        pos2 = BIO_tell(bio);
-        if (pos2 < 0 || pos2 <= pos)
-            break;
-        ossl_clear_error();
-        pos = pos2;
+        }
     }
-
   out:
-    OSSL_DECODER_CTX_free(dctx);
+    if (selections) {
+        free(selections);
+    }
     return pkey;
 }
 #else

ext/openssl/ossl_provider.c

@@ -184,9 +184,20 @@ ossl_provider_inspect(VALUE self)
                       rb_obj_class(self), OSSL_PROVIDER_get0_name(prov));
 }
 
+static int
+count_provider(OSSL_PROVIDER *prov, void *cbdata)
+{
+    int *nump = (int *)cbdata;
+    (*nump)++;
+    /* Always return ok. */
+    return 1;
+}
+#endif
+
 void
 Init_ossl_provider(void)
 {
+#ifdef OSSL_USE_PROVIDER
 #if 0
     mOSSL = rb_define_module("OpenSSL");
     eOSSLError = rb_define_class_under(mOSSL, "OpenSSLError", rb_eStandardError);
@@ -202,10 +213,15 @@ Init_ossl_provider(void)
     rb_define_method(cProvider, "unload", ossl_provider_unload, 0);
     rb_define_method(cProvider, "name", ossl_provider_get_name, 0);
     rb_define_method(cProvider, "inspect", ossl_provider_inspect, 0);
+#endif
 }
-#else
-void
-Init_ossl_provider(void)
+
+int
+ossl_count_provider_number(void)
 {
-}
+    int num = 0;
+#ifdef OSSL_USE_PROVIDER
+    OSSL_PROVIDER_do_all(NULL, &count_provider, (void*)&num);
 #endif
+    return num;
+}

ext/openssl/ossl_provider.h

@@ -2,4 +2,5 @@
 #define OSSL_PROVIDER_H
 
 void Init_ossl_provider(void);
+int ossl_count_provider_number(void);
 #endif