diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs
new file mode 100644
index 0000000000..0a59b34c1e
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs
@@ -0,0 +1,48 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Encryption.Custom
+{
+ using System.IO.Compression;
+
+ ///
+ /// Options for payload compression
+ ///
+ public class CompressionOptions
+ {
+ ///
+ /// Supported compression algorithms
+ ///
+ /// Compression is only supported with .NET8.0+.
+ public enum CompressionAlgorithm
+ {
+ ///
+ /// No compression
+ ///
+ None = 0,
+#if NET8_0_OR_GREATER
+
+ ///
+ /// Brotli compression
+ ///
+ Brotli = 1,
+#endif
+ }
+
+ ///
+ /// Gets or sets compression algorithm.
+ ///
+ public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.None;
+
+ ///
+ /// Gets or sets compression level.
+ ///
+ public CompressionLevel CompressionLevel { get; set; } = CompressionLevel.Fastest;
+
+ ///
+ /// Gets or sets minimal property size for compression.
+ ///
+ public int MinimalCompressedLength { get; set; } = 128;
+ }
+}
\ No newline at end of file
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs
index c59c74cc67..84f4d1b79b 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs
@@ -13,6 +13,8 @@ internal static class Constants
public const string EncryptionDekId = "_en";
public const string EncryptionFormatVersion = "_ef";
public const string EncryptedPaths = "_ep";
+ public const string CompressionAlgorithm = "_ce";
+ public const string CompressedEncryptedPaths = "_cp";
public const int DekPropertiesDefaultTTLInMinutes = 120;
}
}
\ No newline at end of file
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs
index 088965d747..574b07d687 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs
@@ -21,7 +21,7 @@ public static class CosmosEncryptionAlgorithm
/// MDE(Microsoft.Data.Encryption) Randomized AEAD_AES_256_CBC_HMAC_SHA256 Algorithm.
/// As described here.
///
- public const string MdeAeadAes256CbcHmac256Randomized = "MdeAeadAes256CbcHmac256Randomized";
+ public const string MdeAeadAes256CbcHmac256Randomized = @"MdeAeadAes256CbcHmac256Randomized";
///
/// Verify if the Encryption Algorithm is supported by Cosmos.
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs
index a5dff8f12b..163643f0d1 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs
@@ -29,6 +29,11 @@ public sealed class EncryptionOptions
///
public string EncryptionAlgorithm { get; set; }
+ ///
+ /// Gets or sets payload compression mode
+ ///
+ public CompressionOptions CompressionOptions { get; set; } = new CompressionOptions();
+
///
/// Gets or sets list of JSON paths to encrypt on the payload.
/// Only top level paths are supported.
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs
new file mode 100644
index 0000000000..1297652a25
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs
@@ -0,0 +1,47 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Encryption.Custom
+{
+ using System;
+
+ internal static class EncryptionOptionsExtensions
+ {
+ internal static void Validate(this EncryptionOptions options)
+ {
+ if (string.IsNullOrWhiteSpace(options.DataEncryptionKeyId))
+ {
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
+ throw new ArgumentNullException(nameof(options.DataEncryptionKeyId));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
+ }
+
+ if (string.IsNullOrWhiteSpace(options.EncryptionAlgorithm))
+ {
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
+ throw new ArgumentNullException(nameof(options.EncryptionAlgorithm));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
+ }
+
+ if (options.PathsToEncrypt == null)
+ {
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
+ throw new ArgumentNullException(nameof(options.PathsToEncrypt));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
+ }
+
+ options.CompressionOptions?.Validate();
+ }
+
+ internal static void Validate(this CompressionOptions options)
+ {
+ if (options.MinimalCompressedLength < 0)
+ {
+#pragma warning disable CA2208 // Instantiate argument exceptions correctly
+ throw new ArgumentOutOfRangeException(nameof(options.MinimalCompressedLength));
+#pragma warning restore CA2208 // Instantiate argument exceptions correctly
+ }
+ }
+ }
+}
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs
index 53d74390f8..72f77d6700 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs
@@ -206,22 +206,7 @@ private static void ValidateInputForEncrypt(
}
#endif
-#pragma warning disable CA2208 // Instantiate argument exceptions correctly
- if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId))
- {
- throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId));
- }
-
- if (string.IsNullOrWhiteSpace(encryptionOptions.EncryptionAlgorithm))
- {
- throw new ArgumentNullException(nameof(encryptionOptions.EncryptionAlgorithm));
- }
-
- if (encryptionOptions.PathsToEncrypt == null)
- {
- throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt));
- }
-#pragma warning restore CA2208 // Instantiate argument exceptions correctly
+ encryptionOptions.Validate();
}
private static JObject RetrieveItem(
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs
index 8e0ccaf84a..70f385e064 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs
@@ -24,18 +24,28 @@ internal class EncryptionProperties
[JsonProperty(PropertyName = Constants.EncryptedPaths)]
public IEnumerable EncryptedPaths { get; }
+ [JsonProperty(PropertyName = Constants.CompressionAlgorithm)]
+ public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; }
+
+ [JsonProperty(PropertyName = Constants.CompressedEncryptedPaths)]
+ public IDictionary CompressedEncryptedPaths { get; }
+
public EncryptionProperties(
int encryptionFormatVersion,
string encryptionAlgorithm,
string dataEncryptionKeyId,
byte[] encryptedData,
- IEnumerable encryptedPaths)
+ IEnumerable encryptedPaths,
+ CompressionOptions.CompressionAlgorithm compressionAlgorithm = CompressionOptions.CompressionAlgorithm.None,
+ IDictionary compressedEncryptedPaths = null)
{
this.EncryptionFormatVersion = encryptionFormatVersion;
this.EncryptionAlgorithm = encryptionAlgorithm;
this.DataEncryptionKeyId = dataEncryptionKeyId;
this.EncryptedData = encryptedData;
this.EncryptedPaths = encryptedPaths;
+ this.CompressionAlgorithm = compressionAlgorithm;
+ this.CompressedEncryptedPaths = compressedEncryptedPaths;
}
}
}
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs
new file mode 100644
index 0000000000..20f16efc27
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs
@@ -0,0 +1,110 @@
+// ------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation
+{
+#if NET8_0_OR_GREATER
+
+ using System;
+ using System.Buffers;
+ using System.Collections.Generic;
+ using System.IO.Compression;
+
+ internal class BrotliCompressor : IDisposable
+ {
+ private const int DefaultWindow = 22;
+
+ internal static int GetQualityFromCompressionLevel(CompressionLevel compressionLevel)
+ {
+ return compressionLevel switch
+ {
+ CompressionLevel.NoCompression => 0,
+ CompressionLevel.Fastest => 1,
+ CompressionLevel.Optimal => 4,
+ CompressionLevel.SmallestSize => 11,
+ _ => throw new ArgumentException("Unsupported compression level", nameof(compressionLevel))
+ };
+ }
+
+ internal static int GetMaxCompressedSize(int inputSize)
+ {
+ return BrotliEncoder.GetMaxCompressedLength(inputSize);
+ }
+
+ private readonly BrotliDecoder decoder;
+ private readonly BrotliEncoder encoder;
+ private bool disposedValue;
+
+ public BrotliCompressor()
+ {
+ }
+
+ public BrotliCompressor(CompressionLevel compressionLevel)
+ {
+ this.encoder = new BrotliEncoder(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), DefaultWindow);
+ }
+
+ public virtual int Compress(Dictionary compressedPaths, string path, byte[] bytes, int length, byte[] outputBytes)
+ {
+ OperationStatus status = this.encoder.Compress(bytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten, true);
+
+ ThrowIfFailure(status, length, bytesConsumed);
+
+ compressedPaths[path] = length;
+
+ return bytesWritten;
+ }
+
+ public virtual int Decompress(byte[] inputBytes, int length, byte[] outputBytes)
+ {
+ OperationStatus status = this.decoder.Decompress(inputBytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten);
+
+ ThrowIfFailure(status, length, bytesConsumed);
+
+ return bytesWritten;
+ }
+
+ private static void ThrowIfFailure(OperationStatus status, int expectedLength, int processedLength)
+ {
+ if (status != OperationStatus.Done)
+ {
+ throw new InvalidOperationException($"Brotli compressor failed : {status}");
+ }
+
+ if (expectedLength != processedLength)
+ {
+ throw new InvalidOperationException($"Expected to process {expectedLength} but only processed {processedLength}");
+ }
+ }
+
+ protected virtual void Dispose(bool disposing)
+ {
+ if (!this.disposedValue)
+ {
+ if (disposing)
+ {
+ }
+
+ // TODO: free unmanaged resources (unmanaged objects) and override finalizer
+ this.encoder.Dispose();
+
+ this.disposedValue = true;
+ }
+ }
+
+ ~BrotliCompressor()
+ {
+ // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+ this.Dispose(disposing: false);
+ }
+
+ public void Dispose()
+ {
+ // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
+ this.Dispose(disposing: true);
+ GC.SuppressFinalize(this);
+ }
+ }
+#endif
+}
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs
index e4447c3c5c..4786e9f1a4 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs
@@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation
{
using System;
using System.Collections.Generic;
+ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
@@ -34,6 +35,14 @@ public async Task EncryptAsync(
DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token);
+ bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None;
+
+#if NET8_0_OR_GREATER
+ BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli
+ ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null;
+#endif
+ Dictionary compressedPaths = new ();
+
foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt)
{
#if NET8_0_OR_GREATER
@@ -51,26 +60,41 @@ public async Task EncryptAsync(
continue;
}
- byte[] plainText = null;
- (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager);
+ byte[] processedBytes = null;
+ (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager);
- if (plainText == null)
+ if (processedBytes == null)
{
continue;
}
- byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength);
+#if NET8_0_OR_GREATER
+ if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength))
+ {
+ byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength));
+ processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes);
+ processedBytes = compressedBytes;
+ }
+#endif
+
+ byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength);
itemJObj[propertyName] = encryptedBytes;
+
pathsEncrypted.Add(pathToEncrypt);
}
+#if NET8_0_OR_GREATER
+ compressor?.Dispose();
+#endif
EncryptionProperties encryptionProperties = new (
- encryptionFormatVersion: 3,
+ encryptionFormatVersion: compressionEnabled ? 4 : 3,
encryptionOptions.EncryptionAlgorithm,
encryptionOptions.DataEncryptionKeyId,
encryptedData: null,
- pathsEncrypted);
+ pathsEncrypted,
+ encryptionOptions.CompressionOptions.Algorithm,
+ compressedPaths);
itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties));
#if NET8_0_OR_GREATER
@@ -90,7 +114,7 @@ internal async Task DecryptObjectAsync(
{
_ = diagnosticsContext;
- if (encryptionProperties.EncryptionFormatVersion != 3)
+ if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4)
{
throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version.");
}
@@ -101,6 +125,24 @@ internal async Task DecryptObjectAsync(
DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken);
List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count());
+
+#if NET8_0_OR_GREATER
+ BrotliCompressor decompressor = null;
+ if (encryptionProperties.EncryptionFormatVersion == 4)
+ {
+ bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true;
+ if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed)
+ {
+ throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}");
+ }
+
+ if (containsCompressed)
+ {
+ decompressor = new ();
+ }
+ }
+#endif
+
foreach (string path in encryptionProperties.EncryptedPaths)
{
#if NET8_0_OR_GREATER
@@ -121,11 +163,24 @@ internal async Task DecryptObjectAsync(
continue;
}
- (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager);
+ (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager);
+
+#if NET8_0_OR_GREATER
+ if (decompressor != null)
+ {
+ if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true)
+ {
+ byte[] buffer = arrayPoolManager.Rent(decompressedSize);
+ processedBytes = decompressor.Decompress(bytes, processedBytes, buffer);
+
+ bytes = buffer;
+ }
+ }
+#endif
this.Serializer.DeserializeAndAddProperty(
(TypeMarker)cipherTextWithTypeMarker[0],
- plainText.AsSpan(0, decryptedCount),
+ bytes.AsSpan(0, processedBytes),
document,
propertyName,
charPoolManager);
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs
index 638222785f..910817db3d 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs
@@ -25,6 +25,9 @@ public partial class EncryptionBenchmark
[Params(1, 10, 100)]
public int DocumentSizeInKb { get; set; }
+ [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)]
+ public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; }
+
[GlobalSetup]
public async Task Setup()
{
@@ -38,7 +41,7 @@ public async Task Setup()
.ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue));
this.encryptor = new(keyProvider.Object);
- this.encryptionOptions = CreateEncryptionOptions();
+ this.encryptionOptions = this.CreateEncryptionOptions();
this.plaintext = this.LoadTestDoc();
Stream encryptedStream = await EncryptionProcessor.EncryptAsync(
@@ -74,13 +77,17 @@ await EncryptionProcessor.DecryptAsync(
CancellationToken.None);
}
- private static EncryptionOptions CreateEncryptionOptions()
+ private EncryptionOptions CreateEncryptionOptions()
{
EncryptionOptions options = new()
{
DataEncryptionKeyId = "dekId",
EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized,
- PathsToEncrypt = TestDoc.PathsToEncrypt
+ PathsToEncrypt = TestDoc.PathsToEncrypt,
+ CompressionOptions = new()
+ {
+ Algorithm = this.CompressionAlgorithm
+ }
};
return options;
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj
index 821014c014..edcdac998b 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj
@@ -6,6 +6,7 @@
net8.0
enable
enable
+ true
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md
index ced6fc95ed..2913528571 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md
@@ -1,19 +1,25 @@
``` ini
-BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169)
+BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.26100.2033)
11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores
-.NET SDK=8.0.400
- [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2
+.NET SDK=8.0.403
+ [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2
Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15
LaunchCount=2 WarmupCount=10
```
-| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated |
-|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:|
-| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** |
-| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB |
-| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** |
-| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB |
-| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** |
-| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB |
+| Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated |
+|-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:|
+| **Encrypt** | **1** | **None** | **22.45 μs** | **0.447 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **40.94 KB** |
+| Decrypt | 1 | None | 26.65 μs | 0.165 μs | 0.247 μs | 0.1526 | 0.0305 | - | 40.32 KB |
+| **Encrypt** | **1** | **Brotli** | **27.92 μs** | **0.212 μs** | **0.317 μs** | **0.1526** | **0.0305** | **-** | **37.3 KB** |
+| Decrypt | 1 | Brotli | 33.45 μs | 0.718 μs | 1.075 μs | 0.1221 | - | - | 39.95 KB |
+| **Encrypt** | **10** | **None** | **83.56 μs** | **0.358 μs** | **0.502 μs** | **0.6104** | **0.1221** | **-** | **167.12 KB** |
+| Decrypt | 10 | None | 100.88 μs | 0.437 μs | 0.627 μs | 0.6104 | 0.1221 | - | 153.59 KB |
+| **Encrypt** | **10** | **Brotli** | **111.94 μs** | **0.456 μs** | **0.669 μs** | **0.6104** | **0.1221** | **-** | **164.26 KB** |
+| Decrypt | 10 | Brotli | 121.06 μs | 2.794 μs | 4.182 μs | 0.4883 | - | - | 141.31 KB |
+| **Encrypt** | **100** | **None** | **1,194.10 μs** | **37.744 μs** | **56.494 μs** | **23.4375** | **23.4375** | **21.4844** | **1638.78 KB** |
+| Decrypt | 100 | None | 1,247.89 μs | 32.037 μs | 47.952 μs | 17.5781 | 15.6250 | 15.6250 | 1230.56 KB |
+| **Encrypt** | **100** | **Brotli** | **1,199.53 μs** | **30.018 μs** | **44.930 μs** | **13.6719** | **11.7188** | **9.7656** | **1347 KB** |
+| Decrypt | 100 | Brotli | 1,196.64 μs | 22.702 μs | 33.979 μs | 11.7188 | 9.7656 | 9.7656 | 1097.75 KB |
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs
new file mode 100644
index 0000000000..373a120a60
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs
@@ -0,0 +1,56 @@
+namespace Microsoft.Azure.Cosmos.Encryption.Tests
+{
+ using System;
+ using System.Collections.Generic;
+ using Microsoft.Azure.Cosmos.Encryption.Custom;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [TestClass]
+ public class EncryptionOptionsExtensionsTests
+ {
+ [TestMethod]
+ public void Validate_EncryptionOptions_Throws()
+ {
+ Assert.ThrowsException(() => new EncryptionOptions()
+ {
+ DataEncryptionKeyId = null,
+ EncryptionAlgorithm = "something",
+ PathsToEncrypt = new List()
+ }.Validate());
+
+ Assert.ThrowsException(() => new EncryptionOptions()
+ {
+ DataEncryptionKeyId = "something",
+ EncryptionAlgorithm = null,
+ PathsToEncrypt = new List()
+ }.Validate());
+
+ Assert.ThrowsException(() => new EncryptionOptions()
+ {
+ DataEncryptionKeyId = "something",
+ EncryptionAlgorithm = "something",
+ PathsToEncrypt = null
+ }.Validate());
+
+ Assert.ThrowsException(() => new EncryptionOptions()
+ {
+ DataEncryptionKeyId = "something",
+ EncryptionAlgorithm = "something",
+ PathsToEncrypt = new List(),
+ CompressionOptions = new CompressionOptions()
+ {
+ MinimalCompressedLength = -1
+ }
+ }.Validate());
+ }
+
+ [TestMethod]
+ public void Validate_CompressionOptions_Throws()
+ {
+ Assert.ThrowsException(() => new CompressionOptions()
+ {
+ MinimalCompressedLength = -1
+ }.Validate());
+ }
+ }
+}
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs
new file mode 100644
index 0000000000..7d0c2562c2
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs
@@ -0,0 +1,54 @@
+namespace Microsoft.Azure.Cosmos.Encryption.Tests
+{
+ using System;
+ using System.Collections.Generic;
+ using System.Linq;
+ using Microsoft.Azure.Cosmos.Encryption.Custom;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [TestClass]
+ public class EncryptionPropertiesTests
+ {
+ [TestMethod]
+ public void Ctor_AssignsAllMandatoryProperties()
+ {
+ EncryptionProperties properties = new (
+ 11,
+ "algorithm",
+ "dek-id",
+ new byte[] { 1, 2, 3, 4, 5 },
+ new List { "a", "b" });
+
+ Assert.AreEqual(11, properties.EncryptionFormatVersion);
+ Assert.AreEqual("algorithm", properties.EncryptionAlgorithm);
+ Assert.AreEqual("dek-id", properties.DataEncryptionKeyId);
+ Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5}.SequenceEqual(properties.EncryptedData));
+ Assert.IsTrue(new List { "a", "b"}.SequenceEqual(properties.EncryptedPaths));
+ Assert.AreEqual(CompressionOptions.CompressionAlgorithm.None, properties.CompressionAlgorithm);
+ Assert.IsNull(properties.CompressedEncryptedPaths);
+ }
+
+#if NET8_0_OR_GREATER
+ [TestMethod]
+ public void Ctor_AssignsAllProperties()
+ {
+ EncryptionProperties properties = new (
+ 11,
+ "algorithm",
+ "dek-id",
+ new byte[] { 1, 2, 3, 4, 5 },
+ new List { "a", "b" },
+ CompressionOptions.CompressionAlgorithm.Brotli,
+ new Dictionary { { "a", 246 } });
+
+ Assert.AreEqual(11, properties.EncryptionFormatVersion);
+ Assert.AreEqual("algorithm", properties.EncryptionAlgorithm);
+ Assert.AreEqual("dek-id", properties.DataEncryptionKeyId);
+ Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5 }.SequenceEqual(properties.EncryptedData));
+ Assert.IsTrue(new List { "a", "b" }.SequenceEqual(properties.EncryptedPaths));
+ Assert.AreEqual(CompressionOptions.CompressionAlgorithm.Brotli, properties.CompressionAlgorithm);
+ Assert.IsTrue(new Dictionary { { "a", 246 } }.SequenceEqual(properties.CompressedEncryptedPaths));
+ }
+#endif
+ }
+}
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs
index 69a9c7afb0..410a9fa64e 100644
--- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs
@@ -20,19 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests
public class MdeEncryptionProcessorTests
{
private static Mock mockEncryptor;
- private static EncryptionOptions encryptionOptions;
private const string dekId = "dekId";
[ClassInitialize]
public static void ClassInitialize(TestContext testContext)
{
_ = testContext;
- encryptionOptions = new EncryptionOptions()
- {
- DataEncryptionKeyId = dekId,
- EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized,
- PathsToEncrypt = TestDoc.PathsToEncrypt
- };
Mock DekMock = new();
DekMock.Setup(m => m.EncryptData(It.IsAny()))
@@ -125,12 +118,13 @@ await EncryptionProcessor.EncryptAsync(
}
[TestMethod]
- public async Task EncryptDecryptPropertyWithNullValue()
+ [DynamicData(nameof(EncryptionOptionsCombinations))]
+ public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions)
{
TestDoc testDoc = TestDoc.Create();
testDoc.SensitiveStr = null;
- JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc);
+ JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions);
(JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync(
encryptedDoc,
@@ -146,11 +140,12 @@ public async Task EncryptDecryptPropertyWithNullValue()
}
[TestMethod]
- public async Task ValidateEncryptDecryptDocument()
+ [DynamicData(nameof(EncryptionOptionsCombinations))]
+ public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions)
{
TestDoc testDoc = TestDoc.Create();
- JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc);
+ JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions);
(JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync(
encryptedDoc,
@@ -166,7 +161,8 @@ public async Task ValidateEncryptDecryptDocument()
}
[TestMethod]
- public async Task ValidateDecryptStream()
+ [DynamicData(nameof(EncryptionOptionsCombinations))]
+ public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions)
{
TestDoc testDoc = TestDoc.Create();
@@ -209,7 +205,7 @@ public async Task DecryptStreamWithoutEncryptedProperty()
Assert.IsNull(decryptionContext);
}
- private static async Task VerifyEncryptionSucceeded(TestDoc testDoc)
+ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions)
{
Stream encryptedStream = await EncryptionProcessor.EncryptAsync(
testDoc.ToStream(),
@@ -234,7 +230,12 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc)
Assert.IsNotNull(encryptionProperties);
Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId);
- Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion);
+
+ int expectedVersion =
+ (encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None)
+ ? 4 : 3;
+ Assert.AreEqual(expectedVersion, encryptionProperties.EncryptionFormatVersion);
+
Assert.IsNull(encryptionProperties.EncryptedData);
Assert.IsNotNull(encryptionProperties.EncryptedPaths);
@@ -287,5 +288,45 @@ private static void VerifyDecryptionSucceeded(
}
}
}
+
+ public static IEnumerable
+
-
+
-
+
-
+
-
+
-
+
true
true
diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs
new file mode 100644
index 0000000000..d8f5b611dd
--- /dev/null
+++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs
@@ -0,0 +1,75 @@
+namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation
+{
+#if NET8_0_OR_GREATER
+ using System;
+ using System.Collections.Generic;
+ using System.IO.Compression;
+ using System.Linq;
+ using Microsoft.Azure.Cosmos.Encryption.Custom;
+ using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation;
+ using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+ [TestClass]
+ public class BrotliCompressionTests
+ {
+ [TestMethod]
+ [DataRow(CompressionLevel.NoCompression, 0)]
+ [DataRow(CompressionLevel.Fastest, 1)]
+ [DataRow(CompressionLevel.Optimal, 4)]
+ [DataRow(CompressionLevel.SmallestSize, 11)]
+ public void GetQuality_WorksForKnownLevels(CompressionLevel compressionLevel, int expectedQuality)
+ {
+ Assert.AreEqual(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), expectedQuality);
+ }
+
+ [TestMethod]
+ public void GetQuality_ThrowsForUnknownLevels()
+ {
+ Assert.ThrowsException(() => BrotliCompressor.GetQualityFromCompressionLevel((CompressionLevel)(-999)));
+ }
+
+ [TestMethod]
+ [DataRow(CompressionLevel.NoCompression, 256)]
+ [DataRow(CompressionLevel.Fastest, 256)]
+ [DataRow(CompressionLevel.Optimal, 256)]
+ [DataRow(CompressionLevel.SmallestSize, 256)]
+ [DataRow(CompressionLevel.NoCompression, 1024)]
+ [DataRow(CompressionLevel.Fastest, 1024)]
+ [DataRow(CompressionLevel.Optimal, 1024)]
+ [DataRow(CompressionLevel.SmallestSize, 1024)]
+ [DataRow(CompressionLevel.NoCompression, 4096)]
+ [DataRow(CompressionLevel.Fastest, 4096)]
+ [DataRow(CompressionLevel.Optimal, 4096)]
+ [DataRow(CompressionLevel.SmallestSize, 4096)]
+ public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize)
+ {
+ BrotliCompressor compressor = new (compressionLevel);
+ Dictionary properties = new ();
+ string path = "somePath";
+
+ byte[] bytes = new byte[payloadSize];
+ bytes.AsSpan().Fill(127);
+
+ byte[] compressedBytes = new byte[BrotliCompressor.GetMaxCompressedSize(payloadSize)];
+ int compressedBytesSize = compressor.Compress(properties, path, bytes, bytes.Length, compressedBytes);
+
+ Assert.AreNotSame(bytes, compressedBytes);
+ Assert.IsTrue(compressedBytesSize > 0);
+ Assert.IsTrue(compressedBytesSize < bytes.Length);
+
+ Console.WriteLine($"Original: {bytes.Length} Compressed: {compressedBytesSize}");
+
+ Assert.IsTrue(properties.ContainsKey(path));
+
+ int recordedSize = properties["somePath"];
+ Assert.AreEqual(bytes.Length, recordedSize);
+
+ byte[] decompressedBytes = new byte[recordedSize];
+ int decompressedBytesSize = compressor.Decompress(compressedBytes, compressedBytesSize, decompressedBytes);
+
+ Assert.AreEqual(decompressedBytesSize, bytes.Length);
+ Assert.IsTrue(bytes.SequenceEqual(decompressedBytes));
+ }
+ }
+#endif
+}