Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add TryGetValue_False_FrozenDictionary benchmark #3432

Merged
merged 1 commit into from
Nov 29, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ namespace System.Collections
[BenchmarkCategory(Categories.ThirdParty)]
public abstract class Perf_FrozenDictionary_String
{
protected string[] _array;
protected string[] _array, _notFound;
protected Dictionary<string, string> _dictionary;
protected ImmutableDictionary<string, string> _immutableDictionary;
protected FrozenDictionary<string, string> _frozenDictionary;
Expand Down Expand Up @@ -65,12 +65,27 @@ public bool TryGetValue_True_FrozenDictionary()
return result;
}

protected void EnsureRightStrategyIsUsed(string name)
[Benchmark]
[BenchmarkCategory(Categories.Libraries)]
public bool TryGetValue_False_FrozenDictionary()
{
bool result = default;
var collection = _frozenDictionary;
string[] notFound = _notFound;
for (int i = 0; i < notFound.Length; i++)
result ^= collection.TryGetValue(notFound[i], out _);
return result;
}

protected void Verify(string name)
{
if (!_frozenDictionary.GetType().Name.Contains(name))
{
// commented out until https://github.com/dotnet/runtime/pull/87988 gets propagated to installer
// throw new InvalidOperationException($"Either we are using wrong strategy ({_frozenDictionary.GetType().Name}), or the type has been renamed.");
throw new InvalidOperationException($"Either we are using wrong strategy ({_frozenDictionary.GetType().Name}), or the type has been renamed.");
}
else if (_array.Length != _notFound.Length || _array.Any(text => _notFound.Contains(text)))
{
throw new InvalidOperationException("Invalid state");
}
}
}
Expand All @@ -90,14 +105,16 @@ public override void Setup()
throw new ArgumentException($"{nameof(Count)} needs to be a multiply of {nameof(ItemsPerBucket)}");
}

_array = Enumerable.Range(1, Count / ItemsPerBucket)
string[] all = Enumerable.Range(1, Count * 2 / ItemsPerBucket)
.SelectMany(length => Enumerable.Range('a', ItemsPerBucket).Select(character => new string((char)character, length)))
.ToArray();
_array = all.Take(Count).ToArray();
_notFound = all.Skip(Count).ToArray();
_dictionary = _array.ToDictionary(item => item, item => item);
_immutableDictionary = _dictionary.ToImmutableDictionary();
_frozenDictionary = _dictionary.ToFrozenDictionary();

EnsureRightStrategyIsUsed("LengthBucketsFrozenDictionary");
Verify("LengthBucketsFrozenDictionary");
}
}

Expand All @@ -108,14 +125,16 @@ public class Perf_SingleCharFrozenDictionary : Perf_FrozenDictionary_String

public override void Setup()
{
_array = Enumerable.Range(char.MinValue, Count)
string[] all = Enumerable.Range(char.MinValue, Count * 2)
.Select(character => new string((char)character, 10))
.ToArray();
_array = all.Take(Count).ToArray();
_notFound = all.Skip(Count).ToArray();
_dictionary = _array.ToDictionary(item => item, item => item);
_immutableDictionary = _dictionary.ToImmutableDictionary();
_frozenDictionary = _dictionary.ToFrozenDictionary();

EnsureRightStrategyIsUsed("SingleChar");
Verify("SingleChar");
}
}

Expand All @@ -137,18 +156,20 @@ public override void Setup()
// bcbbbbbb
// bdbbbbbb
// so the first char is not unique, but the combination of 1st and 2nd is.
_array = Enumerable.Range(char.MinValue, Count / 2)
string[] all = Enumerable.Range(char.MinValue, Count)
.SelectMany(character => new string[]
{
$"{(char)character}{(char)(character+1)}{new string((char)character, 8)}",
$"{(char)character}{(char)(character+2)}{new string((char)character, 8)}"
})
.ToArray();
_array = all.Take(Count).ToArray();
_notFound = all.Skip(Count).ToArray();
_dictionary = _array.ToDictionary(item => item, item => item);
_immutableDictionary = _dictionary.ToImmutableDictionary();
_frozenDictionary = _dictionary.ToFrozenDictionary();

EnsureRightStrategyIsUsed("Substring");
Verify("Substring");
}
}

Expand All @@ -159,16 +180,17 @@ public class Perf_DefaultFrozenDictionary : Perf_FrozenDictionary_String

public override void Setup()
{
_array = Count == 10
string[] all = Count == 10
// to avoid using LengthBucketsFrozenDictionary we specify the same length for more than 5 strings
? ValuesGenerator.ArrayOfUniqueStrings(10, minLength: 25, maxLength: 25)
: ValuesGenerator.ArrayOfUniqueValues<string>(Count);

? ValuesGenerator.ArrayOfUniqueStrings(10 * 2, minLength: 25, maxLength: 25)

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please consider adding a new case for 20 items designed to benchmark the length optimizations.

                // bimodal distribution of lengths with some search strings outside bounds of the keys
                : Count == 20 ? ValuesGenerator.ArrayOfUniqueStrings(10, minLength: 10, maxLength: 12)
                        .Concat(ValuesGenerator.ArrayOfUniqueStrings(10, minLength: 18, maxLength: 20))
                        .Concat(ValuesGenerator.ArrayOfUniqueStrings(10 * 2, minLength: 5, maxLength: 25)).ToArray()

I ran the benchmarks against 3 versions of .NET runtime.
main - main branch as of yesterday
nolength - I removed the existing if ((uint)(key.Length - _minimumLength) <= (uint)_maximumLengthDiff) check for comparison's sake
pr92546 - dotnet/runtime#92546 which adds a new length filter

Here are the results:

// * Summary *

BenchmarkDotNet v0.13.7-nightly.20230717.35, Windows 11 (10.0.22621.2428/22H2/2022Update/SunValley2)
AMD Ryzen 9 5950X, 1 CPU, 32 logical and 16 physical cores
.NET SDK 8.0.100-rc.1.23463.5
  [Host]     : .NET 8.0.0 (8.0.23.41904), X64 RyuJIT AVX2
  Job-DLBUBA : .NET 9.0.0 (42.42.42.42424), X64 RyuJIT AVX2
  Job-LYUJAP : .NET 9.0.0 (42.42.42.42424), X64 RyuJIT AVX2
  Job-EJCRGQ : .NET 9.0.0 (42.42.42.42424), X64 RyuJIT AVX2

PowerPlanMode=00000000-0000-0000-0000-000000000000  Arguments=/p:EnableUnsafeBinaryFormatterSerialization=true  IterationTime=250.0000 ms  
MaxIterationCount=20  MinIterationCount=15  WarmupCount=1  

|                             Method |        Job |                         ...   Toolchain | Count |          Mean |        Error |       StdDev |        Median |           Min |           Max | Ratio | RatioSD | Allocated | Alloc Ratio |
|----------------------------------- |----------- |-------------------------...------------ |------ |--------------:|-------------:|-------------:|--------------:|--------------:|--------------:|------:|--------:|----------:|------------:|
| TryGetValue_False_FrozenDictionary | Job-DLBUBA |     \dotnet-runtime-main...\corerun.exe |    10 |      22.72 ns |     0.177 ns |     0.165 ns |      22.78 ns |      22.31 ns |      22.89 ns |  1.00 |    0.00 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-LYUJAP | \dotnet-runtime-nolength...\corerun.exe |    10 |      22.04 ns |     0.746 ns |     0.829 ns |      21.67 ns |      21.03 ns |      23.72 ns |  0.97 |    0.04 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-EJCRGQ |  \dotnet-runtime-pr92546...\corerun.exe |    10 |      24.14 ns |     0.305 ns |     0.285 ns |      24.19 ns |      23.61 ns |      24.53 ns |  1.06 |    0.02 |         - |          NA |
|                                    |            |                         ...             |       |               |              |              |               |               |               |       |         |           |             |
| TryGetValue_False_FrozenDictionary | Job-DLBUBA |     \dotnet-runtime-main...\corerun.exe |    20 |      37.76 ns |     0.761 ns |     0.747 ns |      37.59 ns |      36.78 ns |      39.03 ns |  1.00 |    0.00 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-LYUJAP | \dotnet-runtime-nolength...\corerun.exe |    20 |      79.91 ns |     1.291 ns |     1.207 ns |      79.65 ns |      78.42 ns |      82.54 ns |  2.12 |    0.06 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-EJCRGQ |  \dotnet-runtime-pr92546...\corerun.exe |    20 |      21.93 ns |     0.449 ns |     0.441 ns |      21.80 ns |      21.51 ns |      23.26 ns |  0.58 |    0.02 |         - |          NA |
|                                    |            |                         ...             |       |               |              |              |               |               |               |       |         |           |             |
| TryGetValue_False_FrozenDictionary | Job-DLBUBA |     \dotnet-runtime-main...\corerun.exe |   100 |     716.16 ns |     5.014 ns |     4.187 ns |     716.11 ns |     710.24 ns |     725.50 ns |  1.00 |    0.00 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-LYUJAP | \dotnet-runtime-nolength...\corerun.exe |   100 |     686.85 ns |     4.755 ns |     4.448 ns |     686.33 ns |     678.03 ns |     694.17 ns |  0.96 |    0.01 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-EJCRGQ |  \dotnet-runtime-pr92546...\corerun.exe |   100 |     561.36 ns |     3.986 ns |     3.728 ns |     562.77 ns |     555.22 ns |     567.82 ns |  0.78 |    0.01 |         - |          NA |
|                                    |            |                         ...             |       |               |              |              |               |               |               |       |         |           |             |
| TryGetValue_False_FrozenDictionary | Job-DLBUBA |     \dotnet-runtime-main...\corerun.exe |  1000 |   7,837.84 ns |    49.749 ns |    46.535 ns |   7,815.30 ns |   7,781.38 ns |   7,899.49 ns |  1.00 |    0.00 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-LYUJAP | \dotnet-runtime-nolength...\corerun.exe |  1000 |   7,701.66 ns |     7.215 ns |     6.749 ns |   7,700.41 ns |   7,692.06 ns |   7,714.71 ns |  0.98 |    0.01 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-EJCRGQ |  \dotnet-runtime-pr92546...\corerun.exe |  1000 |   7,936.08 ns |    49.304 ns |    46.119 ns |   7,957.89 ns |   7,838.63 ns |   7,987.06 ns |  1.01 |    0.01 |         - |          NA |
|                                    |            |                         ...             |       |               |              |              |               |               |               |       |         |           |             |
| TryGetValue_False_FrozenDictionary | Job-DLBUBA |     \dotnet-runtime-main...\corerun.exe | 10000 | 140,558.00 ns | 4,053.557 ns | 4,668.082 ns | 138,462.17 ns | 135,838.37 ns | 148,890.43 ns |  1.00 |    0.00 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-LYUJAP | \dotnet-runtime-nolength...\corerun.exe | 10000 | 136,547.30 ns | 1,650.734 ns | 1,544.097 ns | 136,678.12 ns | 134,415.09 ns | 139,203.45 ns |  0.97 |    0.04 |         - |          NA |
| TryGetValue_False_FrozenDictionary | Job-EJCRGQ |  \dotnet-runtime-pr92546...\corerun.exe | 10000 | 139,220.39 ns |   850.721 ns |   795.765 ns | 139,317.75 ns | 137,574.00 ns | 140,382.59 ns |  0.99 |    0.04 |         - |          NA |

I believe the new Count = 20 scenario case showcases both the existing optimization (without PR92546), with the ratio going from 2.12 (optimization removed) -> 1 (main), as well as the proposed optimization in PR92546 with the ratio improving from 1 to 0.58.

Although the effects of PR92546 are also visible in the Count=100 scenario, I would argue that testing strings with lengths outside the bounds of the key strings is an important benchmark to keep.

: ValuesGenerator.ArrayOfUniqueValues<string>(Count * 2);
_array = all.Take(Count).ToArray();
_notFound = all.Skip(Count).ToArray();
_dictionary = _array.ToDictionary(item => item, item => item);
_immutableDictionary = _dictionary.ToImmutableDictionary();
_frozenDictionary = _dictionary.ToFrozenDictionary();

EnsureRightStrategyIsUsed("OrdinalStringFrozenDictionary");
Verify("OrdinalStringFrozenDictionary");
}
}
}
Loading