cipher: allow setting IV length when using AEAD ciphers

Add OpenSSL::Cipher#iv_len=. For interoperability with other
applications, it is sometimes required. Normally 'IV' is fixed-length,
but in OpenSSL, some ciphers such as aes-128-gcm make use of it as
'nonce', which is variable-length.

Changing the IV length in Cipher#iv= is also an option but I decided not
to choose it. Because in Ruby <= 2.3 Cipher#iv= truncates the input when
the length is longer than the current IV length, changing the behavior
might cause unexpected encryption result.

[Bug #8667] [Bug #10420] [GH ruby/ruby#569]

NEWS

@@ -49,6 +49,10 @@ Updates since Ruby 2.3
   - OpenSSL::Cipher#key= and #iv= reject too long inputs. They used to truncate
     silently. [Bug #12561]
 
+  - OpenSSL::Cipher#iv_len= is added. It allows changing IV (nonce) length if
+    using AEAD ciphers.
+    [Bug #8667] [Bug #10420] [GH ruby/ruby#569] [GH ruby/openssl#58]
+
 * OpenSSL::Engine
 
   - OpenSSL::Engine.cleanup does nothing when built with OpenSSL 1.1.0.

ext/openssl/openssl_missing.h

@@ -231,6 +231,7 @@ IMPL_PKEY_GETTER(EC_KEY, ec)
 #if defined(HAVE_AUTHENTICATED_ENCRYPTION) && !defined(EVP_CTRL_AEAD_GET_TAG)
 #  define EVP_CTRL_AEAD_GET_TAG EVP_CTRL_GCM_GET_TAG
 #  define EVP_CTRL_AEAD_SET_TAG EVP_CTRL_GCM_SET_TAG
+#  define EVP_CTRL_AEAD_SET_IVLEN EVP_CTRL_GCM_SET_IVLEN
 #endif
 
 #endif /* _OSSL_OPENSSL_MISSING_H_ */

ext/openssl/ossl_cipher.c

@@ -499,12 +499,17 @@ static VALUE
 ossl_cipher_set_iv(VALUE self, VALUE iv)
 {
     EVP_CIPHER_CTX *ctx;
-    int iv_len;
+    int iv_len = 0;
 
     StringValue(iv);
     GetCipher(self, ctx);
 
-    iv_len = EVP_CIPHER_CTX_iv_length(ctx);
+#if defined(HAVE_AUTHENTICATED_ENCRYPTION)
+    if (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER)
+	iv_len = (int)(VALUE)EVP_CIPHER_CTX_get_app_data(ctx);
+#endif
+    if (!iv_len)
+	iv_len = EVP_CIPHER_CTX_iv_length(ctx);
     if (RSTRING_LEN(iv) != iv_len)
 	ossl_raise(rb_eArgError, "iv must be %d bytes", iv_len);
 
@@ -638,11 +643,42 @@ ossl_cipher_is_authenticated(VALUE self)
 
     return (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER) ? Qtrue : Qfalse;
 }
+
+/*
+ * call-seq:
+ *   cipher.iv_len = integer -> integer
+ *
+ * Sets the IV/nonce length of the Cipher. Normally block ciphers don't allow
+ * changing the IV length, but some make use of IV for 'nonce'. You may need
+ * this for interoperability with other applications.
+ */
+static VALUE
+ossl_cipher_set_iv_length(VALUE self, VALUE iv_length)
+{
+    int len = NUM2INT(iv_length);
+    EVP_CIPHER_CTX *ctx;
+
+    GetCipher(self, ctx);
+    if (!(EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER))
+	ossl_raise(eCipherError, "cipher does not support AEAD");
+
+    if (!EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_AEAD_SET_IVLEN, len, NULL))
+	ossl_raise(eCipherError, "unable to set IV length");
+
+    /*
+     * EVP_CIPHER_CTX_iv_length() returns the default length. So we need to save
+     * the length somewhere. Luckily currently we aren't using app_data.
+     */
+    EVP_CIPHER_CTX_set_app_data(ctx, (void *)(VALUE)len);
+
+    return iv_length;
+}
 #else
 #define ossl_cipher_set_auth_data rb_f_notimplement
 #define ossl_cipher_get_auth_tag rb_f_notimplement
 #define ossl_cipher_set_auth_tag rb_f_notimplement
 #define ossl_cipher_is_authenticated rb_f_notimplement
+#define ossl_cipher_set_iv_length rb_f_notimplement
 #endif
 
 /*
@@ -708,13 +744,30 @@ ossl_cipher_set_padding(VALUE self, VALUE padding)
  *  Returns the key length in bytes of the Cipher.
  */
 CIPHER_0ARG_INT(key_length)
+
 /*
  *  call-seq:
  *     cipher.iv_len -> integer
  *
  *  Returns the expected length in bytes for an IV for this Cipher.
  */
-CIPHER_0ARG_INT(iv_length)
+static VALUE
+ossl_cipher_iv_length(VALUE self)
+{
+    EVP_CIPHER_CTX *ctx;
+    int len = 0;
+
+    GetCipher(self, ctx);
+#if defined(HAVE_AUTHENTICATED_ENCRYPTION)
+    if (EVP_CIPHER_CTX_flags(ctx) & EVP_CIPH_FLAG_AEAD_CIPHER)
+	len = (int)(VALUE)EVP_CIPHER_CTX_get_app_data(ctx);
+#endif
+    if (!len)
+	len = EVP_CIPHER_CTX_iv_length(ctx);
+
+    return INT2NUM(len);
+}
+
 /*
  *  call-seq:
  *     cipher.block_size -> integer
@@ -952,6 +1005,7 @@ Init_ossl_cipher(void)
     rb_define_method(cCipher, "key_len=", ossl_cipher_set_key_length, 1);
     rb_define_method(cCipher, "key_len", ossl_cipher_key_length, 0);
     rb_define_method(cCipher, "iv=", ossl_cipher_set_iv, 1);
+    rb_define_method(cCipher, "iv_len=", ossl_cipher_set_iv_length, 1);
     rb_define_method(cCipher, "iv_len", ossl_cipher_iv_length, 0);
     rb_define_method(cCipher, "block_size", ossl_cipher_block_size, 0);
     rb_define_method(cCipher, "padding=", ossl_cipher_set_padding, 1);

test/test_cipher.rb

@@ -247,6 +247,46 @@ def test_aes_gcm_wrong_ciphertext
       end
     end
 
+    def test_aes_gcm_variable_iv_len
+      pt = "You should all use Authenticated Encryption!"
+      cipher = OpenSSL::Cipher.new("aes-128-gcm").encrypt
+      cipher.key = "x" * 16
+      assert_equal(12, cipher.iv_len)
+      cipher.iv = "a" * 12
+      ct1 = cipher.update(pt) << cipher.final
+      tag1 = cipher.auth_tag
+
+      cipher = OpenSSL::Cipher.new("aes-128-gcm").encrypt
+      cipher.key = "x" * 16
+      cipher.iv_len = 10
+      assert_equal(10, cipher.iv_len)
+      cipher.iv = "a" * 10
+      ct2 = cipher.update(pt) << cipher.final
+      tag2 = cipher.auth_tag
+
+      assert_not_equal ct1, ct2
+      assert_not_equal tag1, tag2
+
+      decipher = OpenSSL::Cipher.new("aes-128-gcm").decrypt
+      decipher.auth_tag = tag1
+      decipher.key = "x" * 16
+      decipher.iv_len = 12
+      decipher.iv = "a" * 12
+      assert_equal(pt, decipher.update(ct1) << decipher.final)
+
+      decipher.reset
+      decipher.auth_tag = tag2
+      assert_raise(OpenSSL::Cipher::CipherError) {
+        decipher.update(ct2) << decipher.final
+      }
+
+      decipher.reset
+      decipher.auth_tag = tag2
+      decipher.iv_len = 10
+      decipher.iv = "a" * 10
+      assert_equal(pt, decipher.update(ct2) << decipher.final)
+    end
+
   end
 
   private