Add new features to MongoDbStorage provider (MiniProfiler#613)

* New features:
    * Decimal fields can now be serialized as `NumberDecimal`s instead of raw strings.
    * Indexes can be automatically created instead of having to manually call the `WithIndexCreation` method.
    * MiniProfiler sessions can be configured to be automatically expired (deleted) after a certain time period has elapsed.
* The `MongoDbStorageOptions` class has been added to allow for setting the above options.
* MongoDB C# driver has been updated to the latest version, and obsoleted code elements updated accordingly.
* All new features are opt-in; full backwards-compatibility (including binary) is retained.

Co-authored-by: Ian Kemp <ian.kemp@capitalontap.com>
Co-authored-by: Nick Craver <nrcraver@gmail.com>

appveyor.yml

@@ -37,7 +37,6 @@ services:
   - mssql2019
   - mysql
   - postgresql
-  - mongodb
 
 nuget:
   disable_publish_on_pr: true

docs/Releases.md

@@ -6,7 +6,8 @@ layout: "default"
 This page tracks major changes included in any update starting with version 4.0.0.3
 
 #### Unreleased
-No pending unreleased changes.
+- **Fixes/Changes**: 
+  - Upgraded MongoDB driver, allowing automatic index creation and profiler expiration ([#613](https://github.com/MiniProfiler/dotnet/pull/613) - thanks [IanKemp](https://github.com/IanKemp))
 
 #### Version 4.3.8
 - **New**:

src/MiniProfiler.Providers.MongoDB/MiniProfiler.Providers.MongoDB.csproj

@@ -3,14 +3,14 @@
     <AssemblyName>MiniProfiler.Providers.MongoDB</AssemblyName>
     <Title>MiniProfiler.Providers.MongoDB</Title>
     <Description>MiniProfiler: Profiler storage for MongoDB</Description>
-    <Authors>Nick Craver, Roger Calaf</Authors>
+    <Authors>Nick Craver, Roger Calaf, Ian Kemp</Authors>
     <PackageTags>NoSQL;MongoDB;$(PackageBaseTags)</PackageTags>
     <TargetFrameworks>net461;netstandard2.0</TargetFrameworks>
     <AssemblyOriginatorKeyFile>..\..\miniprofiler.snk</AssemblyOriginatorKeyFile>
     <SignAssembly>false</SignAssembly>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\MiniProfiler.Shared\MiniProfiler.Shared.csproj" />
-    <PackageReference Include="MongoDB.Driver" Version="2.5.0" />
+    <PackageReference Include="MongoDB.Driver" Version="2.20.0" />
   </ItemGroup>
 </Project>

src/MiniProfiler.Providers.MongoDB/MongoDbStorage.cs

@@ -1,8 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
+using MongoDB.Bson;
 using MongoDB.Bson.Serialization;
+using MongoDB.Bson.Serialization.Serializers;
 using MongoDB.Driver;
+using MongoDB.Driver.Core.Operations;
 using StackExchange.Profiling.Storage;
 
 namespace StackExchange.Profiling
@@ -12,38 +16,69 @@ namespace StackExchange.Profiling
     /// </summary>
     public class MongoDbStorage : IAsyncStorage
     {
+        private readonly MongoDbStorageOptions _options;
         private readonly MongoClient _client;
         private readonly IMongoCollection<MiniProfiler> _collection;
 
         /// <summary>
         /// Returns a new <see cref="MongoDbStorage"/>. MongoDb connection string will default to "mongodb://localhost"
+        /// and collection name to "profilers".
         /// </summary>
         /// <param name="connectionString">The MongoDB connection string.</param>
         public MongoDbStorage(string connectionString) : this(connectionString, "profilers") { }
 
         /// <summary>
-        /// Returns a new <see cref="MongoDbStorage"/>. MongoDb connection string will default to "mongodb://localhost"
+        /// Returns a new <see cref="MongoDbStorage"/>. MongoDb connection string will default to "mongodb://localhost".
         /// </summary>
         /// <param name="connectionString">The MongoDB connection string.</param>
         /// <param name="collectionName">The collection name to use in the database.</param>
-        public MongoDbStorage(string connectionString, string collectionName)
+        public MongoDbStorage(string connectionString, string collectionName) : this(new MongoDbStorageOptions
+        {
+           ConnectionString = connectionString,
+           CollectionName = collectionName,
+        }) { }
+
+        /// <summary>
+        /// Creates a new instance of this class using the provided <paramref name="options"/>.
+        /// </summary>
+        /// <param name="options">Options to use for configuring this instance.</param>
+        /// <exception cref="ArgumentException">If <see cref="MongoDbStorageOptions.CollectionName"/> is null or contains only whitespace.</exception>
+        public MongoDbStorage(MongoDbStorageOptions options)
         {
+            if (string.IsNullOrWhiteSpace(options.CollectionName))
+            {
+                throw new ArgumentException("Collection name may not be null or contain only whitespace", nameof(options.CollectionName));
+            }
+
+            _options = options;
+
             if (!BsonClassMap.IsClassMapRegistered(typeof(MiniProfiler)))
             {
                 BsonClassMapFields();
             }
 
-            var url = new MongoUrl(connectionString);
+            var url = new MongoUrl(options.ConnectionString);
             var databaseName = url.DatabaseName ?? "MiniProfiler";
 
             _client = new MongoClient(url);
             _collection = _client
                 .GetDatabase(databaseName)
-                .GetCollection<MiniProfiler>(collectionName);
+                .GetCollection<MiniProfiler>(options.CollectionName);
+
+            if (options.AutomaticallyCreateIndexes)
+            {
+                WithIndexCreation(options.CacheDuration);
+            }
         }
 
-        private static void BsonClassMapFields()
+        private void BsonClassMapFields()
         {
+            if (_options.SerializeDecimalFieldsAsNumberDecimal)
+            {
+                BsonSerializer.RegisterSerializer(typeof(decimal), new DecimalSerializer(BsonType.Decimal128));
+                BsonSerializer.RegisterSerializer(typeof(decimal?), new NullableSerializer<decimal>(new DecimalSerializer(BsonType.Decimal128)));
+            }
+
             BsonClassMap.RegisterClassMap<MiniProfiler>(
                 map =>
                 {
@@ -97,13 +132,59 @@ private static void BsonClassMapFields()
         /// </summary>
         public MongoDbStorage WithIndexCreation()
         {
-            _collection.Indexes.CreateOne(Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.User));
-            _collection.Indexes.CreateOne(Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.HasUserViewed));
-            _collection.Indexes.CreateOne(Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.Started));
-            _collection.Indexes.CreateOne(Builders<MiniProfiler>.IndexKeys.Descending(_ => _.Started));
+            _collection.Indexes.CreateOne(new CreateIndexModel<MiniProfiler>(Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.User)));
+            _collection.Indexes.CreateOne(new CreateIndexModel<MiniProfiler>(Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.HasUserViewed)));
+            CreateStartedAscendingIndex();
+            _collection.Indexes.CreateOne(new CreateIndexModel<MiniProfiler>(Builders<MiniProfiler>.IndexKeys.Descending(_ => _.Started)));
+
             return this;
         }
 
+        /// <summary>
+        /// Creates indexes on the following fields for faster querying:
+        /// <list type="table">
+        /// <listheader><term>Field</term><term>Direction</term><term>Notes</term></listheader>
+        /// <item><term>User</term><term>Ascending</term><term></term></item>
+        /// <item><term>HasUserViewed</term><term>Ascending</term><term></term></item>
+        /// <item><term>Started</term><term>Ascending</term><term>Used to apply the <paramref name="cacheDuration"/>, if one was specified</term></item>
+        /// <item><term>Started</term><term>Descending</term><term></term></item>
+        /// </list>
+        /// </summary>
+        /// <param name="cacheDuration">The time to persist profiles before they expire.</param>
+        public MongoDbStorage WithIndexCreation(TimeSpan cacheDuration)
+        {
+            _options.CacheDuration = cacheDuration;
+            return WithIndexCreation();
+        }
+
+        private void CreateStartedAscendingIndex()
+        {
+            var index = Builders<MiniProfiler>.IndexKeys.Ascending(_ => _.Started);
+            var options = _options.CacheDuration != default
+                ? new CreateIndexOptions { ExpireAfter = _options.CacheDuration }
+                : null;
+            var model = new CreateIndexModel<MiniProfiler>(index, options);
+
+            try
+            {
+                _collection.Indexes.CreateOne(model);
+            }
+            catch (MongoCommandException ex) when (_options.AutomaticallyRecreateIndexes && ex.Code == 85)
+            {
+                // Handling the case we found an conflicting existing index, and were told to re-create if this happens
+                var indexNames = _collection.Indexes.List().ToList()
+                                                    .SelectMany(index => index.Elements)
+                                                    .Where(element => element.Name == "name")
+                                                    .Select(name => name.Value.ToString());
+                var indexName = IndexNameHelper.GetIndexName(model.Keys.Render(_collection.Indexes.DocumentSerializer, _collection.Indexes.Settings.SerializerRegistry));
+                if (indexNames.Contains(indexName))
+                {
+                    _collection.Indexes.DropOne(indexName);
+                }
+                _collection.Indexes.CreateOne(model);
+            }
+        }
+
         /// <summary>
         /// Returns a list of <see cref="MiniProfiler.Id"/>s that haven't been seen by <paramref name="user"/>.
         /// </summary>
@@ -188,7 +269,7 @@ public void Save(MiniProfiler profiler)
             _collection.ReplaceOne(
                 p => p.Id == profiler.Id,
                 profiler,
-                new UpdateOptions
+                new ReplaceOptions
                 {
                     IsUpsert = true
                 });
@@ -203,7 +284,7 @@ public Task SaveAsync(MiniProfiler profiler)
             return _collection.ReplaceOneAsync(
                 p => p.Id == profiler.Id,
                 profiler,
-                new UpdateOptions
+                new ReplaceOptions
                 {
                     IsUpsert = true
                 });

src/MiniProfiler.Providers.MongoDB/MongoDbStorageOptions.cs

@@ -0,0 +1,55 @@
+using System;
+
+namespace StackExchange.Profiling
+{
+    /// <summary>
+    /// Options for configuring <see cref="MongoDbStorage"/>.
+    /// </summary>
+    public class MongoDbStorageOptions
+    {
+        /// <summary>
+        /// The connection string to use for connecting to MongoDB.
+        /// Defaults to <c>mongodb://localhost</c>.
+        /// </summary>
+        public string? ConnectionString { get; set; }
+
+        /// <summary>
+        /// Name of the collection in which to store <see cref="MiniProfiler"/> sessions in.
+        /// Defaults to <c>profilers</c>.
+        /// </summary>
+        public string CollectionName { get; set; } = "profilers";
+
+        /// <summary>
+        /// If set to <see langword="true"/>, C# <c>decimal</c> fields will be serialized as <c>NumberDecimal</c>s in MongoDB.
+        /// If set to <see langword="false"/>, will serialize these fields as strings (backwards-compatible with older versions of this provider).
+        /// Defaults to <see langword="true"/>.
+        /// </summary>
+        public bool SerializeDecimalFieldsAsNumberDecimal { get; set; } = true;
+
+        /// <summary>
+        /// Specifies whether relevant indexes will automatically created when this provider is instantiated.
+        /// Defaults to <see langword="true" />.
+        /// </summary>
+        public bool AutomaticallyCreateIndexes { get; set; } = true;
+
+        /// <summary>
+        /// Specifies whether relevant indexes will automatically recreated if creation fails (e.g. because something with
+        /// different options was previously created).
+        /// *THIS DROPS EXISTING DATA*
+        /// Defaults to <see langword="false" />.
+        /// </summary>
+        public bool AutomaticallyRecreateIndexes { get; set; } = false;
+
+        /// <summary>
+        /// Gets or sets how long to cache each <see cref="MiniProfiler"/> for, in absolute terms.
+        /// Defaults to one hour.
+        /// </summary>
+        /// <remarks><list type="bullet">
+        /// <item>You need to either set <see cref="AutomaticallyCreateIndexes"/> to true or call
+        /// <see cref="MongoDbStorage.WithIndexCreation(TimeSpan)"/> for this value to have any effect.</item>
+        /// <item>Setting this option will drop any (<see cref="MiniProfiler.Started"/>, ascending) index previously
+        /// defined, including those with custom options.</item>
+        /// </list></remarks>
+        public TimeSpan CacheDuration { get; set; } = TimeSpan.FromHours(1);
+    }
+}

tests/MiniProfiler.Tests/Helpers/TestConfig.cs

@@ -30,7 +30,7 @@ static TestConfig()
         public class Config
         {
             public bool RunLongRunning { get; set; }
-            public bool EnableTestLogging { get; set; } = Environment.GetEnvironmentVariable(nameof(EnableTestLogging)) == "true";
+            public bool EnableTestLogging { get; set; } = bool.TryParse(Environment.GetEnvironmentVariable(nameof(EnableTestLogging)), out var enableTestLogging) && enableTestLogging;
 
             public string RedisConnectionString { get; set; } = Environment.GetEnvironmentVariable(nameof(RedisConnectionString)) ?? "localhost:6379";
             public string SQLServerConnectionString { get; set; } = Environment.GetEnvironmentVariable(nameof(SQLServerConnectionString)) ?? "Server=.;Database=tempdb;Trusted_Connection=True;";

tests/MiniProfiler.Tests/Storage/MongoDbStorageTests.cs

@@ -1,4 +1,5 @@
 using System;
+using MongoDB.Driver;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -9,9 +10,37 @@ public class MongoDbStorageTests : StorageBaseTest, IClassFixture<MongoDbStorage
         public MongoDbStorageTests(MongoDbStorageFixture fixture, ITestOutputHelper output) : base(fixture, output)
         {
         }
+
+        [Fact]
+        public void RecreationHandling()
+        {
+            var options = new MongoDbStorageOptions
+            {
+                ConnectionString = TestConfig.Current.MongoDbConnectionString,
+                CollectionName = "MPTest" + Guid.NewGuid().ToString("N").Substring(20),
+            };
+
+            var storage = new MongoDbStorage(options);
+            Assert.NotNull(storage);
+            // Same options, won't throw
+            var storage2 = new MongoDbStorage(options);
+            Assert.NotNull(storage2);
+
+            options.CacheDuration = TimeSpan.FromSeconds(20);
+
+            // MongoDB.Driver.MongoCommandException : Command createIndexes failed: Index with name: Started_1 already exists with different options/An equivalent index already exists with the same name but different options.
+            var ex = Assert.Throws<MongoCommandException>(() => new MongoDbStorage(options));
+            Assert.NotNull(ex);
+            Assert.Equal(85, ex.Code);
+
+            options.AutomaticallyRecreateIndexes = true;
+            // Succeeds, because drop/re-create is allowed now
+            var storage4 = new MongoDbStorage(options);
+            Assert.NotNull(storage4);
+        }
     }
 
-    public class MongoDbStorageFixture : StorageFixtureBase<MongoDbStorage>, IDisposable
+    public class MongoDbStorageFixture : StorageFixtureBase<MongoDbStorage>
     {
         public MongoDbStorageFixture()
         {

tests/MiniProfiler.Tests/Storage/RavenDbStoreTests.cs

@@ -30,31 +30,33 @@ public RavenDbStoreFixture()
             {
                 var store = new DocumentStore
                 {
-                    Urls = TestConfig.Current.RavenDbUrls.Split(';'), Database = TestConfig.Current.RavenDatabase
+                    Urls = TestConfig.Current.RavenDbUrls.Split(';'),
+                    Database = TestConfig.Current.RavenDatabase + TestId
                 };
 
                 store.Initialize();
 
                 try
                 {
-                    store.Maintenance.ForDatabase(TestConfig.Current.RavenDatabase).Send(new GetStatisticsOperation());
+                    store.Maintenance.ForDatabase(store.Database).Send(new GetStatisticsOperation());
                 }
                 catch (DatabaseDoesNotExistException)
                 {
                     try
                     {
-                        store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(TestConfig.Current.RavenDatabase)));
+                        store.Maintenance.Server.Send(new CreateDatabaseOperation(new DatabaseRecord(store.Database)));
                     }
                     catch (ConcurrencyException)
                     {
                         // The database was already created before calling CreateDatabaseOperation
                     }
                 }
 
+                var dbName = store.Database;
                 store.Dispose();
                 store = null;
 
-                Storage = new RavenDbStorage(TestConfig.Current.RavenDbUrls.Split(';'), TestConfig.Current.RavenDatabase, waitForIndexes: true);
+                Storage = new RavenDbStorage(TestConfig.Current.RavenDbUrls.Split(';'), dbName, waitForIndexes: true);
                 Storage.GetUnviewedIds("");
             }
             catch (Exception e)