diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs new file mode 100644 index 0000000000..0a59b34c1e --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CompressionOptions.cs @@ -0,0 +1,48 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System.IO.Compression; + + /// + /// Options for payload compression + /// + public class CompressionOptions + { + /// + /// Supported compression algorithms + /// + /// Compression is only supported with .NET8.0+. + public enum CompressionAlgorithm + { + /// + /// No compression + /// + None = 0, +#if NET8_0_OR_GREATER + + /// + /// Brotli compression + /// + Brotli = 1, +#endif + } + + /// + /// Gets or sets compression algorithm. + /// + public CompressionAlgorithm Algorithm { get; set; } = CompressionAlgorithm.None; + + /// + /// Gets or sets compression level. + /// + public CompressionLevel CompressionLevel { get; set; } = CompressionLevel.Fastest; + + /// + /// Gets or sets minimal property size for compression. + /// + public int MinimalCompressedLength { get; set; } = 128; + } +} \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs index c59c74cc67..84f4d1b79b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Constants.cs @@ -13,6 +13,8 @@ internal static class Constants public const string EncryptionDekId = "_en"; public const string EncryptionFormatVersion = "_ef"; public const string EncryptedPaths = "_ep"; + public const string CompressionAlgorithm = "_ce"; + public const string CompressedEncryptedPaths = "_cp"; public const int DekPropertiesDefaultTTLInMinutes = 120; } } \ No newline at end of file diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs index 088965d747..574b07d687 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/CosmosEncryptionAlgorithm.cs @@ -21,7 +21,7 @@ public static class CosmosEncryptionAlgorithm /// MDE(Microsoft.Data.Encryption) Randomized AEAD_AES_256_CBC_HMAC_SHA256 Algorithm. /// As described here. /// - public const string MdeAeadAes256CbcHmac256Randomized = "MdeAeadAes256CbcHmac256Randomized"; + public const string MdeAeadAes256CbcHmac256Randomized = @"MdeAeadAes256CbcHmac256Randomized"; /// /// Verify if the Encryption Algorithm is supported by Cosmos. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs index a5dff8f12b..163643f0d1 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptions.cs @@ -29,6 +29,11 @@ public sealed class EncryptionOptions /// public string EncryptionAlgorithm { get; set; } + /// + /// Gets or sets payload compression mode + /// + public CompressionOptions CompressionOptions { get; set; } = new CompressionOptions(); + /// /// Gets or sets list of JSON paths to encrypt on the payload. /// Only top level paths are supported. diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs new file mode 100644 index 0000000000..1297652a25 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionOptionsExtensions.cs @@ -0,0 +1,47 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom +{ + using System; + + internal static class EncryptionOptionsExtensions + { + internal static void Validate(this EncryptionOptions options) + { + if (string.IsNullOrWhiteSpace(options.DataEncryptionKeyId)) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.DataEncryptionKeyId)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + if (string.IsNullOrWhiteSpace(options.EncryptionAlgorithm)) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.EncryptionAlgorithm)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + if (options.PathsToEncrypt == null) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentNullException(nameof(options.PathsToEncrypt)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + + options.CompressionOptions?.Validate(); + } + + internal static void Validate(this CompressionOptions options) + { + if (options.MinimalCompressedLength < 0) + { +#pragma warning disable CA2208 // Instantiate argument exceptions correctly + throw new ArgumentOutOfRangeException(nameof(options.MinimalCompressedLength)); +#pragma warning restore CA2208 // Instantiate argument exceptions correctly + } + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs index 53d74390f8..72f77d6700 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProcessor.cs @@ -206,22 +206,7 @@ private static void ValidateInputForEncrypt( } #endif -#pragma warning disable CA2208 // Instantiate argument exceptions correctly - if (string.IsNullOrWhiteSpace(encryptionOptions.DataEncryptionKeyId)) - { - throw new ArgumentNullException(nameof(encryptionOptions.DataEncryptionKeyId)); - } - - if (string.IsNullOrWhiteSpace(encryptionOptions.EncryptionAlgorithm)) - { - throw new ArgumentNullException(nameof(encryptionOptions.EncryptionAlgorithm)); - } - - if (encryptionOptions.PathsToEncrypt == null) - { - throw new ArgumentNullException(nameof(encryptionOptions.PathsToEncrypt)); - } -#pragma warning restore CA2208 // Instantiate argument exceptions correctly + encryptionOptions.Validate(); } private static JObject RetrieveItem( diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs index 8e0ccaf84a..70f385e064 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/EncryptionProperties.cs @@ -24,18 +24,28 @@ internal class EncryptionProperties [JsonProperty(PropertyName = Constants.EncryptedPaths)] public IEnumerable EncryptedPaths { get; } + [JsonProperty(PropertyName = Constants.CompressionAlgorithm)] + public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; } + + [JsonProperty(PropertyName = Constants.CompressedEncryptedPaths)] + public IDictionary CompressedEncryptedPaths { get; } + public EncryptionProperties( int encryptionFormatVersion, string encryptionAlgorithm, string dataEncryptionKeyId, byte[] encryptedData, - IEnumerable encryptedPaths) + IEnumerable encryptedPaths, + CompressionOptions.CompressionAlgorithm compressionAlgorithm = CompressionOptions.CompressionAlgorithm.None, + IDictionary compressedEncryptedPaths = null) { this.EncryptionFormatVersion = encryptionFormatVersion; this.EncryptionAlgorithm = encryptionAlgorithm; this.DataEncryptionKeyId = dataEncryptionKeyId; this.EncryptedData = encryptedData; this.EncryptedPaths = encryptedPaths; + this.CompressionAlgorithm = compressionAlgorithm; + this.CompressedEncryptedPaths = compressedEncryptedPaths; } } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs new file mode 100644 index 0000000000..20f16efc27 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/BrotliCompressor.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// ------------------------------------------------------------ + +namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation +{ +#if NET8_0_OR_GREATER + + using System; + using System.Buffers; + using System.Collections.Generic; + using System.IO.Compression; + + internal class BrotliCompressor : IDisposable + { + private const int DefaultWindow = 22; + + internal static int GetQualityFromCompressionLevel(CompressionLevel compressionLevel) + { + return compressionLevel switch + { + CompressionLevel.NoCompression => 0, + CompressionLevel.Fastest => 1, + CompressionLevel.Optimal => 4, + CompressionLevel.SmallestSize => 11, + _ => throw new ArgumentException("Unsupported compression level", nameof(compressionLevel)) + }; + } + + internal static int GetMaxCompressedSize(int inputSize) + { + return BrotliEncoder.GetMaxCompressedLength(inputSize); + } + + private readonly BrotliDecoder decoder; + private readonly BrotliEncoder encoder; + private bool disposedValue; + + public BrotliCompressor() + { + } + + public BrotliCompressor(CompressionLevel compressionLevel) + { + this.encoder = new BrotliEncoder(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), DefaultWindow); + } + + public virtual int Compress(Dictionary compressedPaths, string path, byte[] bytes, int length, byte[] outputBytes) + { + OperationStatus status = this.encoder.Compress(bytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten, true); + + ThrowIfFailure(status, length, bytesConsumed); + + compressedPaths[path] = length; + + return bytesWritten; + } + + public virtual int Decompress(byte[] inputBytes, int length, byte[] outputBytes) + { + OperationStatus status = this.decoder.Decompress(inputBytes.AsSpan(0, length), outputBytes, out int bytesConsumed, out int bytesWritten); + + ThrowIfFailure(status, length, bytesConsumed); + + return bytesWritten; + } + + private static void ThrowIfFailure(OperationStatus status, int expectedLength, int processedLength) + { + if (status != OperationStatus.Done) + { + throw new InvalidOperationException($"Brotli compressor failed : {status}"); + } + + if (expectedLength != processedLength) + { + throw new InvalidOperationException($"Expected to process {expectedLength} but only processed {processedLength}"); + } + } + + protected virtual void Dispose(bool disposing) + { + if (!this.disposedValue) + { + if (disposing) + { + } + + // TODO: free unmanaged resources (unmanaged objects) and override finalizer + this.encoder.Dispose(); + + this.disposedValue = true; + } + } + + ~BrotliCompressor() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: false); + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +#endif +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs index e4447c3c5c..4786e9f1a4 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/src/Transformation/MdeEncryptionProcessor.Preview.cs @@ -8,6 +8,7 @@ namespace Microsoft.Azure.Cosmos.Encryption.Custom.Transformation { using System; using System.Collections.Generic; + using System.Diagnostics; using System.IO; using System.Linq; using System.Threading; @@ -34,6 +35,14 @@ public async Task EncryptAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionOptions.DataEncryptionKeyId, encryptionOptions.EncryptionAlgorithm, token); + bool compressionEnabled = encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None; + +#if NET8_0_OR_GREATER + BrotliCompressor compressor = encryptionOptions.CompressionOptions.Algorithm == CompressionOptions.CompressionAlgorithm.Brotli + ? new BrotliCompressor(encryptionOptions.CompressionOptions.CompressionLevel) : null; +#endif + Dictionary compressedPaths = new (); + foreach (string pathToEncrypt in encryptionOptions.PathsToEncrypt) { #if NET8_0_OR_GREATER @@ -51,26 +60,41 @@ public async Task EncryptAsync( continue; } - byte[] plainText = null; - (typeMarker, plainText, int plainTextLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); + byte[] processedBytes = null; + (typeMarker, processedBytes, int processedBytesLength) = this.Serializer.Serialize(propertyValue, arrayPoolManager); - if (plainText == null) + if (processedBytes == null) { continue; } - byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, plainText, plainTextLength); +#if NET8_0_OR_GREATER + if (compressor != null && (processedBytesLength >= encryptionOptions.CompressionOptions.MinimalCompressedLength)) + { + byte[] compressedBytes = arrayPoolManager.Rent(BrotliCompressor.GetMaxCompressedSize(processedBytesLength)); + processedBytesLength = compressor.Compress(compressedPaths, pathToEncrypt, processedBytes, processedBytesLength, compressedBytes); + processedBytes = compressedBytes; + } +#endif + + byte[] encryptedBytes = this.Encryptor.Encrypt(encryptionKey, typeMarker, processedBytes, processedBytesLength); itemJObj[propertyName] = encryptedBytes; + pathsEncrypted.Add(pathToEncrypt); } +#if NET8_0_OR_GREATER + compressor?.Dispose(); +#endif EncryptionProperties encryptionProperties = new ( - encryptionFormatVersion: 3, + encryptionFormatVersion: compressionEnabled ? 4 : 3, encryptionOptions.EncryptionAlgorithm, encryptionOptions.DataEncryptionKeyId, encryptedData: null, - pathsEncrypted); + pathsEncrypted, + encryptionOptions.CompressionOptions.Algorithm, + compressedPaths); itemJObj.Add(Constants.EncryptedInfo, JObject.FromObject(encryptionProperties)); #if NET8_0_OR_GREATER @@ -90,7 +114,7 @@ internal async Task DecryptObjectAsync( { _ = diagnosticsContext; - if (encryptionProperties.EncryptionFormatVersion != 3) + if (encryptionProperties.EncryptionFormatVersion != 3 && encryptionProperties.EncryptionFormatVersion != 4) { throw new NotSupportedException($"Unknown encryption format version: {encryptionProperties.EncryptionFormatVersion}. Please upgrade your SDK to the latest version."); } @@ -101,6 +125,24 @@ internal async Task DecryptObjectAsync( DataEncryptionKey encryptionKey = await encryptor.GetEncryptionKeyAsync(encryptionProperties.DataEncryptionKeyId, encryptionProperties.EncryptionAlgorithm, cancellationToken); List pathsDecrypted = new (encryptionProperties.EncryptedPaths.Count()); + +#if NET8_0_OR_GREATER + BrotliCompressor decompressor = null; + if (encryptionProperties.EncryptionFormatVersion == 4) + { + bool containsCompressed = encryptionProperties.CompressedEncryptedPaths?.Any() == true; + if (encryptionProperties.CompressionAlgorithm != CompressionOptions.CompressionAlgorithm.Brotli && containsCompressed) + { + throw new NotSupportedException($"Unknown compression algorithm {encryptionProperties.CompressionAlgorithm}"); + } + + if (containsCompressed) + { + decompressor = new (); + } + } +#endif + foreach (string path in encryptionProperties.EncryptedPaths) { #if NET8_0_OR_GREATER @@ -121,11 +163,24 @@ internal async Task DecryptObjectAsync( continue; } - (byte[] plainText, int decryptedCount) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + (byte[] bytes, int processedBytes) = this.Encryptor.Decrypt(encryptionKey, cipherTextWithTypeMarker, arrayPoolManager); + +#if NET8_0_OR_GREATER + if (decompressor != null) + { + if (encryptionProperties.CompressedEncryptedPaths?.TryGetValue(path, out int decompressedSize) == true) + { + byte[] buffer = arrayPoolManager.Rent(decompressedSize); + processedBytes = decompressor.Decompress(bytes, processedBytes, buffer); + + bytes = buffer; + } + } +#endif this.Serializer.DeserializeAndAddProperty( (TypeMarker)cipherTextWithTypeMarker[0], - plainText.AsSpan(0, decryptedCount), + bytes.AsSpan(0, processedBytes), document, propertyName, charPoolManager); diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs index 638222785f..910817db3d 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/EncryptionBenchmark.cs @@ -25,6 +25,9 @@ public partial class EncryptionBenchmark [Params(1, 10, 100)] public int DocumentSizeInKb { get; set; } + [Params(CompressionOptions.CompressionAlgorithm.None, CompressionOptions.CompressionAlgorithm.Brotli)] + public CompressionOptions.CompressionAlgorithm CompressionAlgorithm { get; set; } + [GlobalSetup] public async Task Setup() { @@ -38,7 +41,7 @@ public async Task Setup() .ReturnsAsync(() => new MdeEncryptionAlgorithm(DekProperties, EncryptionType.Randomized, StoreProvider.Object, cacheTimeToLive: TimeSpan.MaxValue)); this.encryptor = new(keyProvider.Object); - this.encryptionOptions = CreateEncryptionOptions(); + this.encryptionOptions = this.CreateEncryptionOptions(); this.plaintext = this.LoadTestDoc(); Stream encryptedStream = await EncryptionProcessor.EncryptAsync( @@ -74,13 +77,17 @@ await EncryptionProcessor.DecryptAsync( CancellationToken.None); } - private static EncryptionOptions CreateEncryptionOptions() + private EncryptionOptions CreateEncryptionOptions() { EncryptionOptions options = new() { DataEncryptionKeyId = "dekId", EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new() + { + Algorithm = this.CompressionAlgorithm + } }; return options; diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj index 821014c014..edcdac998b 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests.csproj @@ -6,6 +6,7 @@ net8.0 enable enable + true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md index ced6fc95ed..2913528571 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Performance.Tests/Readme.md @@ -1,19 +1,25 @@ ``` ini -BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.22631.4169) +BenchmarkDotNet=v0.13.3, OS=Windows 11 (10.0.26100.2033) 11th Gen Intel Core i9-11950H 2.60GHz, 1 CPU, 16 logical and 8 physical cores -.NET SDK=8.0.400 - [Host] : .NET 6.0.33 (6.0.3324.36610), X64 RyuJIT AVX2 +.NET SDK=8.0.403 + [Host] : .NET 8.0.10 (8.0.1024.46610), X64 RyuJIT AVX2 Job=MediumRun Toolchain=InProcessEmitToolchain IterationCount=15 LaunchCount=2 WarmupCount=10 ``` -| Method | DocumentSizeInKb | Mean | Error | StdDev | Median | Gen0 | Gen1 | Gen2 | Allocated | -|-------- |----------------- |------------:|-----------:|-----------:|------------:|---------:|---------:|---------:|-----------:| -| **Encrypt** | **1** | **28.40 μs** | **0.428 μs** | **0.640 μs** | **28.40 μs** | **3.3569** | **0.8240** | **-** | **41.15 KB** | -| Decrypt | 1 | 33.19 μs | 0.532 μs | 0.779 μs | 33.54 μs | 3.2349 | 0.7935 | - | 39.7 KB | -| **Encrypt** | **10** | **105.95 μs** | **2.230 μs** | **3.337 μs** | **106.49 μs** | **13.7939** | **0.6104** | **-** | **169.78 KB** | -| Decrypt | 10 | 113.47 μs | 1.716 μs | 2.569 μs | 111.81 μs | 12.5732 | 1.2207 | - | 154.62 KB | -| **Encrypt** | **100** | **1,486.58 μs** | **389.596 μs** | **583.129 μs** | **1,487.32 μs** | **216.7969** | **177.7344** | **142.5781** | **1655.2 KB** | -| Decrypt | 100 | 1,404.48 μs | 137.824 μs | 206.288 μs | 1,409.23 μs | 144.5313 | 107.4219 | 87.8906 | 1248.31 KB | +| Method | DocumentSizeInKb | CompressionAlgorithm | Mean | Error | StdDev | Gen0 | Gen1 | Gen2 | Allocated | +|-------- |----------------- |--------------------- |------------:|----------:|----------:|--------:|--------:|--------:|-----------:| +| **Encrypt** | **1** | **None** | **22.45 μs** | **0.447 μs** | **0.655 μs** | **0.1526** | **0.0305** | **-** | **40.94 KB** | +| Decrypt | 1 | None | 26.65 μs | 0.165 μs | 0.247 μs | 0.1526 | 0.0305 | - | 40.32 KB | +| **Encrypt** | **1** | **Brotli** | **27.92 μs** | **0.212 μs** | **0.317 μs** | **0.1526** | **0.0305** | **-** | **37.3 KB** | +| Decrypt | 1 | Brotli | 33.45 μs | 0.718 μs | 1.075 μs | 0.1221 | - | - | 39.95 KB | +| **Encrypt** | **10** | **None** | **83.56 μs** | **0.358 μs** | **0.502 μs** | **0.6104** | **0.1221** | **-** | **167.12 KB** | +| Decrypt | 10 | None | 100.88 μs | 0.437 μs | 0.627 μs | 0.6104 | 0.1221 | - | 153.59 KB | +| **Encrypt** | **10** | **Brotli** | **111.94 μs** | **0.456 μs** | **0.669 μs** | **0.6104** | **0.1221** | **-** | **164.26 KB** | +| Decrypt | 10 | Brotli | 121.06 μs | 2.794 μs | 4.182 μs | 0.4883 | - | - | 141.31 KB | +| **Encrypt** | **100** | **None** | **1,194.10 μs** | **37.744 μs** | **56.494 μs** | **23.4375** | **23.4375** | **21.4844** | **1638.78 KB** | +| Decrypt | 100 | None | 1,247.89 μs | 32.037 μs | 47.952 μs | 17.5781 | 15.6250 | 15.6250 | 1230.56 KB | +| **Encrypt** | **100** | **Brotli** | **1,199.53 μs** | **30.018 μs** | **44.930 μs** | **13.6719** | **11.7188** | **9.7656** | **1347 KB** | +| Decrypt | 100 | Brotli | 1,196.64 μs | 22.702 μs | 33.979 μs | 11.7188 | 9.7656 | 9.7656 | 1097.75 KB | diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs new file mode 100644 index 0000000000..373a120a60 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionOptionsExtensionsTests.cs @@ -0,0 +1,56 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using System; + using System.Collections.Generic; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class EncryptionOptionsExtensionsTests + { + [TestMethod] + public void Validate_EncryptionOptions_Throws() + { + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = null, + EncryptionAlgorithm = "something", + PathsToEncrypt = new List() + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = null, + PathsToEncrypt = new List() + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = "something", + PathsToEncrypt = null + }.Validate()); + + Assert.ThrowsException(() => new EncryptionOptions() + { + DataEncryptionKeyId = "something", + EncryptionAlgorithm = "something", + PathsToEncrypt = new List(), + CompressionOptions = new CompressionOptions() + { + MinimalCompressedLength = -1 + } + }.Validate()); + } + + [TestMethod] + public void Validate_CompressionOptions_Throws() + { + Assert.ThrowsException(() => new CompressionOptions() + { + MinimalCompressedLength = -1 + }.Validate()); + } + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs new file mode 100644 index 0000000000..7d0c2562c2 --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/EncryptionPropertiesTests.cs @@ -0,0 +1,54 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests +{ + using System; + using System.Collections.Generic; + using System.Linq; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class EncryptionPropertiesTests + { + [TestMethod] + public void Ctor_AssignsAllMandatoryProperties() + { + EncryptionProperties properties = new ( + 11, + "algorithm", + "dek-id", + new byte[] { 1, 2, 3, 4, 5 }, + new List { "a", "b" }); + + Assert.AreEqual(11, properties.EncryptionFormatVersion); + Assert.AreEqual("algorithm", properties.EncryptionAlgorithm); + Assert.AreEqual("dek-id", properties.DataEncryptionKeyId); + Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5}.SequenceEqual(properties.EncryptedData)); + Assert.IsTrue(new List { "a", "b"}.SequenceEqual(properties.EncryptedPaths)); + Assert.AreEqual(CompressionOptions.CompressionAlgorithm.None, properties.CompressionAlgorithm); + Assert.IsNull(properties.CompressedEncryptedPaths); + } + +#if NET8_0_OR_GREATER + [TestMethod] + public void Ctor_AssignsAllProperties() + { + EncryptionProperties properties = new ( + 11, + "algorithm", + "dek-id", + new byte[] { 1, 2, 3, 4, 5 }, + new List { "a", "b" }, + CompressionOptions.CompressionAlgorithm.Brotli, + new Dictionary { { "a", 246 } }); + + Assert.AreEqual(11, properties.EncryptionFormatVersion); + Assert.AreEqual("algorithm", properties.EncryptionAlgorithm); + Assert.AreEqual("dek-id", properties.DataEncryptionKeyId); + Assert.IsTrue(new byte[] { 1, 2, 3, 4, 5 }.SequenceEqual(properties.EncryptedData)); + Assert.IsTrue(new List { "a", "b" }.SequenceEqual(properties.EncryptedPaths)); + Assert.AreEqual(CompressionOptions.CompressionAlgorithm.Brotli, properties.CompressionAlgorithm); + Assert.IsTrue(new Dictionary { { "a", 246 } }.SequenceEqual(properties.CompressedEncryptedPaths)); + } +#endif + } +} diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs index 69a9c7afb0..410a9fa64e 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/MdeEncryptionProcessorTests.cs @@ -20,19 +20,12 @@ namespace Microsoft.Azure.Cosmos.Encryption.Tests public class MdeEncryptionProcessorTests { private static Mock mockEncryptor; - private static EncryptionOptions encryptionOptions; private const string dekId = "dekId"; [ClassInitialize] public static void ClassInitialize(TestContext testContext) { _ = testContext; - encryptionOptions = new EncryptionOptions() - { - DataEncryptionKeyId = dekId, - EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, - PathsToEncrypt = TestDoc.PathsToEncrypt - }; Mock DekMock = new(); DekMock.Setup(m => m.EncryptData(It.IsAny())) @@ -125,12 +118,13 @@ await EncryptionProcessor.EncryptAsync( } [TestMethod] - public async Task EncryptDecryptPropertyWithNullValue() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task EncryptDecryptPropertyWithNullValue(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); testDoc.SensitiveStr = null; - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -146,11 +140,12 @@ public async Task EncryptDecryptPropertyWithNullValue() } [TestMethod] - public async Task ValidateEncryptDecryptDocument() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateEncryptDecryptDocument(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); - JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc); + JObject encryptedDoc = await VerifyEncryptionSucceeded(testDoc, encryptionOptions); (JObject decryptedDoc, DecryptionContext decryptionContext) = await EncryptionProcessor.DecryptAsync( encryptedDoc, @@ -166,7 +161,8 @@ public async Task ValidateEncryptDecryptDocument() } [TestMethod] - public async Task ValidateDecryptStream() + [DynamicData(nameof(EncryptionOptionsCombinations))] + public async Task ValidateDecryptStream(EncryptionOptions encryptionOptions) { TestDoc testDoc = TestDoc.Create(); @@ -209,7 +205,7 @@ public async Task DecryptStreamWithoutEncryptedProperty() Assert.IsNull(decryptionContext); } - private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) + private static async Task VerifyEncryptionSucceeded(TestDoc testDoc, EncryptionOptions encryptionOptions) { Stream encryptedStream = await EncryptionProcessor.EncryptAsync( testDoc.ToStream(), @@ -234,7 +230,12 @@ private static async Task VerifyEncryptionSucceeded(TestDoc testDoc) Assert.IsNotNull(encryptionProperties); Assert.AreEqual(dekId, encryptionProperties.DataEncryptionKeyId); - Assert.AreEqual(3, encryptionProperties.EncryptionFormatVersion); + + int expectedVersion = + (encryptionOptions.CompressionOptions.Algorithm != CompressionOptions.CompressionAlgorithm.None) + ? 4 : 3; + Assert.AreEqual(expectedVersion, encryptionProperties.EncryptionFormatVersion); + Assert.IsNull(encryptionProperties.EncryptedData); Assert.IsNotNull(encryptionProperties.EncryptedPaths); @@ -287,5 +288,45 @@ private static void VerifyDecryptionSucceeded( } } } + + public static IEnumerable EncryptionOptionsCombinations => new[] { + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.None + } + } + }, +#if NET8_0_OR_GREATER + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.Fastest + } + } + }, + new object[] { new EncryptionOptions() + { + DataEncryptionKeyId = dekId, + EncryptionAlgorithm = CosmosEncryptionAlgorithm.MdeAeadAes256CbcHmac256Randomized, + PathsToEncrypt = TestDoc.PathsToEncrypt, + CompressionOptions = new CompressionOptions() + { + Algorithm = CompressionOptions.CompressionAlgorithm.Brotli, + CompressionLevel = System.IO.Compression.CompressionLevel.NoCompression, + } + } + } +#endif + }; } } diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj index 462492ee73..55f611c774 100644 --- a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests.csproj @@ -1,4 +1,4 @@ - + true @@ -9,7 +9,7 @@ false Microsoft.Azure.Cosmos.Encryption.Tests $(LangVersion) - $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW + $(DefineConstants);ENCRYPTION_CUSTOM_PREVIEW @@ -31,21 +31,21 @@ - - - + + + - + - + - + - + - + true true diff --git a/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs new file mode 100644 index 0000000000..d8f5b611dd --- /dev/null +++ b/Microsoft.Azure.Cosmos.Encryption.Custom/tests/Microsoft.Azure.Cosmos.Encryption.Custom.Tests/Transformation/BrotliCompressionTests.cs @@ -0,0 +1,75 @@ +namespace Microsoft.Azure.Cosmos.Encryption.Tests.Transformation +{ +#if NET8_0_OR_GREATER + using System; + using System.Collections.Generic; + using System.IO.Compression; + using System.Linq; + using Microsoft.Azure.Cosmos.Encryption.Custom; + using Microsoft.Azure.Cosmos.Encryption.Custom.Transformation; + using Microsoft.VisualStudio.TestTools.UnitTesting; + + [TestClass] + public class BrotliCompressionTests + { + [TestMethod] + [DataRow(CompressionLevel.NoCompression, 0)] + [DataRow(CompressionLevel.Fastest, 1)] + [DataRow(CompressionLevel.Optimal, 4)] + [DataRow(CompressionLevel.SmallestSize, 11)] + public void GetQuality_WorksForKnownLevels(CompressionLevel compressionLevel, int expectedQuality) + { + Assert.AreEqual(BrotliCompressor.GetQualityFromCompressionLevel(compressionLevel), expectedQuality); + } + + [TestMethod] + public void GetQuality_ThrowsForUnknownLevels() + { + Assert.ThrowsException(() => BrotliCompressor.GetQualityFromCompressionLevel((CompressionLevel)(-999))); + } + + [TestMethod] + [DataRow(CompressionLevel.NoCompression, 256)] + [DataRow(CompressionLevel.Fastest, 256)] + [DataRow(CompressionLevel.Optimal, 256)] + [DataRow(CompressionLevel.SmallestSize, 256)] + [DataRow(CompressionLevel.NoCompression, 1024)] + [DataRow(CompressionLevel.Fastest, 1024)] + [DataRow(CompressionLevel.Optimal, 1024)] + [DataRow(CompressionLevel.SmallestSize, 1024)] + [DataRow(CompressionLevel.NoCompression, 4096)] + [DataRow(CompressionLevel.Fastest, 4096)] + [DataRow(CompressionLevel.Optimal, 4096)] + [DataRow(CompressionLevel.SmallestSize, 4096)] + public void CompressAndDecompress_HasSameResult(CompressionLevel compressionLevel, int payloadSize) + { + BrotliCompressor compressor = new (compressionLevel); + Dictionary properties = new (); + string path = "somePath"; + + byte[] bytes = new byte[payloadSize]; + bytes.AsSpan().Fill(127); + + byte[] compressedBytes = new byte[BrotliCompressor.GetMaxCompressedSize(payloadSize)]; + int compressedBytesSize = compressor.Compress(properties, path, bytes, bytes.Length, compressedBytes); + + Assert.AreNotSame(bytes, compressedBytes); + Assert.IsTrue(compressedBytesSize > 0); + Assert.IsTrue(compressedBytesSize < bytes.Length); + + Console.WriteLine($"Original: {bytes.Length} Compressed: {compressedBytesSize}"); + + Assert.IsTrue(properties.ContainsKey(path)); + + int recordedSize = properties["somePath"]; + Assert.AreEqual(bytes.Length, recordedSize); + + byte[] decompressedBytes = new byte[recordedSize]; + int decompressedBytesSize = compressor.Decompress(compressedBytes, compressedBytesSize, decompressedBytes); + + Assert.AreEqual(decompressedBytesSize, bytes.Length); + Assert.IsTrue(bytes.SequenceEqual(decompressedBytes)); + } + } +#endif +}