Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Memoize Keccaks #7333

Closed
wants to merge 6 commits into from
Closed

Memoize Keccaks #7333

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9845360
Keccak memoize
benaadams Aug 15, 2024
1de1eeb
Reduce ClockCache contention
benaadams Aug 15, 2024
efc9c2c
More variance in Bytes GetHashCode
benaadams Aug 15, 2024
4d15fa7
Feedback
benaadams Aug 15, 2024
597c7bd
Also do storage keys
benaadams Aug 16, 2024
3bc29e7
formatting
benaadams Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/Nethermind/Nethermind.Core/Caching/ClockCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;

using Nethermind.Core.Threading;
Expand Down Expand Up @@ -74,7 +75,8 @@ private bool SetSlow(TKey key, TValue val)
return false;
}

int offset = _cacheMap.Count;
int offset = _count;
Debug.Assert(_cacheMap.Count == _count);
if (FreeOffsets.Count > 0)
{
offset = FreeOffsets.Dequeue();
Expand All @@ -86,14 +88,16 @@ private bool SetSlow(TKey key, TValue val)

_cacheMap[key] = new LruCacheItem(offset, val);
KeyToOffset[offset] = key;
_count++;
LukaszRozmej marked this conversation as resolved.
Show resolved Hide resolved
Debug.Assert(_cacheMap.Count == _count);

return true;
}

private int Replace(TKey key)
{
int position = Clock;
int max = _cacheMap.Count;
int max = _count;
while (true)
{
if (position >= max)
Expand All @@ -108,6 +112,7 @@ private int Replace(TKey key)
{
ThrowInvalidOperationException();
}
_count--;
break;
}

Expand All @@ -132,6 +137,7 @@ public bool Delete(TKey key)

if (_cacheMap.Remove(key, out LruCacheItem? ov))
{
_count--;
KeyToOffset[ov.Offset] = default;
ClearAccessed(ov.Offset);
FreeOffsets.Enqueue(ov.Offset);
Expand All @@ -157,7 +163,7 @@ public bool Contains(TKey key)
return _cacheMap.ContainsKey(key);
}

public int Count => _cacheMap.Count;
public int Count => _count;

private class LruCacheItem(int offset, TValue v)
{
Expand Down
3 changes: 3 additions & 0 deletions src/Nethermind/Nethermind.Core/Caching/ClockCacheBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ public abstract class ClockCacheBase<TKey>
protected Queue<int> FreeOffsets { get; } = new();

protected int Clock { get; set; } = 0;
// Use local count to avoid lock contention with reads on ConcurrentDictionary.Count
protected int _count = 0;

protected ClockCacheBase(int maxCapacity)
{
Expand All @@ -35,6 +37,7 @@ protected void Clear()
{
if (MaxCapacity == 0) return;

_count = 0;
Clock = 0;
FreeOffsets.Clear();
KeyToOffset.AsSpan().Clear();
Expand Down
12 changes: 9 additions & 3 deletions src/Nethermind/Nethermind.Core/Caching/ClockKeyCache.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;

using Nethermind.Core.Threading;
Expand Down Expand Up @@ -51,7 +52,8 @@ private bool SetSlow(TKey key)
return false;
}

offset = _cacheMap.Count;
offset = _count;
Debug.Assert(_cacheMap.Count == _count);
if (FreeOffsets.Count > 0)
{
offset = FreeOffsets.Dequeue();
Expand All @@ -63,14 +65,16 @@ private bool SetSlow(TKey key)

_cacheMap[key] = offset;
KeyToOffset[offset] = key;
_count++;
Debug.Assert(_cacheMap.Count == _count);

return true;
}

private int Replace(TKey key)
{
int position = Clock;
int max = _cacheMap.Count;
int max = _count;
while (true)
{
if (position >= max)
Expand All @@ -85,6 +89,7 @@ private int Replace(TKey key)
{
ThrowInvalidOperationException();
}
_count--;
break;
}

Expand All @@ -108,6 +113,7 @@ public bool Delete(TKey key)

if (_cacheMap.Remove(key, out int offset))
{
_count--;
ClearAccessed(offset);
FreeOffsets.Enqueue(offset);
return true;
Expand All @@ -131,5 +137,5 @@ public bool Contains(TKey key)
return _cacheMap.ContainsKey(key);
}

public int Count => _cacheMap.Count;
public int Count => _count;
}
90 changes: 80 additions & 10 deletions src/Nethermind/Nethermind.Core/Crypto/Keccak.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,20 @@
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Nethermind.Core.Caching;
using Nethermind.Core.Extensions;

namespace Nethermind.Core.Crypto
{
[DebuggerStepThrough]
[DebuggerDisplay("{ToString()}")]
public static class ValueKeccak
{
private static readonly HashCache _cache = new();
/// <returns>
/// <string>0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470</string>
/// </returns>
public static readonly ValueHash256 OfAnEmptyString = InternalCompute(Array.Empty<byte>());
public static readonly ValueHash256 OfAnEmptyString = new("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470");

/// <returns>
/// <string>0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347</string>
Expand Down Expand Up @@ -48,6 +51,17 @@ public static ValueHash256 Compute(string input)
return InternalCompute(System.Text.Encoding.UTF8.GetBytes(input));
}

private readonly struct BytesWrapper(byte[] bytes) : IEquatable<BytesWrapper>
{
internal readonly byte[] _bytes = bytes;
LukaszRozmej marked this conversation as resolved.
Show resolved Hide resolved
public static implicit operator BytesWrapper(byte[] bytes) => new(bytes);
public static implicit operator BytesWrapper(ReadOnlySpan<byte> bytes) => new(bytes.ToArray());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we search by Span without allocating first, like in SpanDictionary?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not in the regular ConcurrentDictionary; though it should live an die pretty fast in Gen0.

Also another reason to limiting to 32 bytes

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Well it is good I made SpanConcurrentDictionary ]:->

public bool Equals(BytesWrapper other) => Bytes.EqualityComparer.Equals(_bytes, other._bytes);
public override bool Equals(object? obj) => obj is BytesWrapper other && Equals(other);
public override int GetHashCode() => Bytes.EqualityComparer.GetHashCode(_bytes);
}

[SkipLocalsInit]
[DebuggerStepThrough]
public static ValueHash256 Compute(ReadOnlySpan<byte> input)
{
Expand All @@ -56,16 +70,72 @@ public static ValueHash256 Compute(ReadOnlySpan<byte> input)
return OfAnEmptyString;
}

Unsafe.SkipInit(out ValueHash256 keccak);
KeccakHash.ComputeHashBytesToSpan(input, MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref keccak, 1)));
return keccak;
Unsafe.SkipInit(out ValueHash256 hash);
Unsafe.SkipInit(out BytesWrapper cacheKey);
if (input.Length <= 32)
LukaszRozmej marked this conversation as resolved.
Show resolved Hide resolved
{
cacheKey = input;
if (_cache.TryGet(cacheKey, out hash)) return hash;
}

KeccakHash.ComputeHashBytesToSpan(input, MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref hash, 1)));
benaadams marked this conversation as resolved.
Show resolved Hide resolved
if (input.Length <= 32)
{
_cache.Set(cacheKey, hash);
}
return hash;
}

internal static ValueHash256 InternalCompute(byte[] input)
{
Unsafe.SkipInit(out ValueHash256 keccak);
KeccakHash.ComputeHashBytesToSpan(input, MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref keccak, 1)));
return keccak;
if (input is null || input.Length == 0)
{
return OfAnEmptyString;
}

Unsafe.SkipInit(out ValueHash256 hash);
if (input.Length <= 32 && _cache.TryGet(input, out hash))
{
return hash;
}

KeccakHash.ComputeHashBytesToSpan(input, MemoryMarshal.AsBytes(MemoryMarshal.CreateSpan(ref hash, 1)));
if (input.Length <= 32)
{
_cache.Set(input, hash);
}
return hash;
}

private sealed class HashCache
{
private const int CacheCount = 16;
benaadams marked this conversation as resolved.
Show resolved Hide resolved
private const int CacheMax = CacheCount - 1;
private readonly ClockCache<BytesWrapper, ValueHash256>[] _caches;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quite a lot of objects here 😢


public HashCache()
{
_caches = new ClockCache<BytesWrapper, ValueHash256>[CacheCount];
for (int i = 0; i < _caches.Length; i++)
{
// Cache per nibble to reduce contention as is very parallel
_caches[i] = new ClockCache<BytesWrapper, ValueHash256>(4096);
}
}

public bool Set(BytesWrapper bytes, in ValueHash256 hash)
{
ClockCache<BytesWrapper, ValueHash256> cache = _caches[GetCacheIndex(bytes)];
return cache.Set(bytes, hash);
}

private static int GetCacheIndex(BytesWrapper bytes) => bytes._bytes[^1] & CacheMax;

public bool TryGet(BytesWrapper bytes, out ValueHash256 hash)
{
ClockCache<BytesWrapper, ValueHash256> cache = _caches[GetCacheIndex(bytes)];
return cache.TryGet(bytes, out hash);
}
}
}

Expand All @@ -82,17 +152,17 @@ public static class Keccak
/// <returns>
/// <string>0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470</string>
/// </returns>
public static readonly Hash256 OfAnEmptyString = new Hash256(ValueKeccak.InternalCompute(Array.Empty<byte>()));
public static readonly Hash256 OfAnEmptyString = new(ValueKeccak.OfAnEmptyString);

/// <returns>
/// <string>0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347</string>
/// </returns>
public static readonly Hash256 OfAnEmptySequenceRlp = new Hash256(ValueKeccak.InternalCompute([192]));
public static readonly Hash256 OfAnEmptySequenceRlp = new(ValueKeccak.OfAnEmptySequenceRlp);

/// <summary>
/// 0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421
/// </summary>
public static Hash256 EmptyTreeHash = new Hash256(ValueKeccak.InternalCompute([128]));
public static Hash256 EmptyTreeHash = new(ValueKeccak.EmptyTreeHash);

/// <returns>
/// <string>0x0000000000000000000000000000000000000000000000000000000000000000</string>
Expand Down
54 changes: 33 additions & 21 deletions src/Nethermind/Nethermind.Core/Extensions/Bytes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,23 +23,25 @@ namespace Nethermind.Core.Extensions
{
public static unsafe partial class Bytes
{
public static readonly IEqualityComparer<byte[]> EqualityComparer = new BytesEqualityComparer();
private static readonly uint s_instanceRandom = (uint)System.Security.Cryptography.RandomNumberGenerator.GetInt32(int.MinValue, int.MaxValue);

public static readonly BytesEqualityComparer EqualityComparer = new BytesEqualityComparer();
public static readonly IEqualityComparer<byte[]?> NullableEqualityComparer = new NullableBytesEqualityComparer();
public static readonly ISpanEqualityComparer<byte> SpanEqualityComparer = new SpanBytesEqualityComparer();
public static readonly BytesComparer Comparer = new();
public static readonly ReadOnlyMemory<byte> ZeroByte = new byte[] { 0 };
public static readonly ReadOnlyMemory<byte> OneByte = new byte[] { 1 };

private class BytesEqualityComparer : EqualityComparer<byte[]>
public class BytesEqualityComparer : EqualityComparer<byte[]>
LukaszRozmej marked this conversation as resolved.
Show resolved Hide resolved
{
public override bool Equals(byte[]? x, byte[]? y)
{
return AreEqual(x, y);
}

public override int GetHashCode(byte[] obj)
public override int GetHashCode(byte[] bytes)
{
return obj.GetSimplifiedHashCode();
return bytes.GetSimplifiedHashCode();
}
}

Expand All @@ -50,17 +52,17 @@ public override bool Equals(byte[]? x, byte[]? y)
return AreEqual(x, y);
}

public override int GetHashCode(byte[]? obj)
public override int GetHashCode(byte[]? bytes)
{
return obj?.GetSimplifiedHashCode() ?? 0;
return bytes?.GetSimplifiedHashCode() ?? 0;
}
}

private class SpanBytesEqualityComparer : ISpanEqualityComparer<byte>
{
public bool Equals(ReadOnlySpan<byte> x, ReadOnlySpan<byte> y) => AreEqual(x, y);

public int GetHashCode(ReadOnlySpan<byte> obj) => GetSimplifiedHashCode(obj);
public int GetHashCode(ReadOnlySpan<byte> bytes) => GetSimplifiedHashCode(bytes);
}

public class BytesComparer : Comparer<byte[]>
Expand Down Expand Up @@ -1124,28 +1126,38 @@ private static byte[] FromHexString(ReadOnlySpan<char> hexString)

[SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")]
public static int GetSimplifiedHashCode(this byte[] bytes)
{
const int fnvPrime = 0x01000193;
=> GetSimplifiedHashCode(bytes.AsSpan());

if (bytes.Length == 0)
[SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")]
public static int GetSimplifiedHashCode(this ReadOnlySpan<byte> span)
{
if (span.Length == 0)
{
return 0;
}

return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53);
}

[SuppressMessage("ReSharper", "NonReadonlyMemberInGetHashCode")]
public static int GetSimplifiedHashCode(this ReadOnlySpan<byte> bytes)
{
const int fnvPrime = 0x01000193;

if (bytes.Length == 0)
uint crc = s_instanceRandom;
var longSize = span.Length / sizeof(ulong) * sizeof(ulong);
if (longSize > 0)
{
return 0;
foreach (ulong ul in MemoryMarshal.Cast<byte, ulong>(span[..longSize]))
{
crc = BitOperations.Crc32C(crc, ul);
}
foreach (byte b in span[longSize..])
{
crc = BitOperations.Crc32C(crc, b);
}
}
else
{
foreach (byte b in span)
{
crc = BitOperations.Crc32C(crc, b);
}
LukaszRozmej marked this conversation as resolved.
Show resolved Hide resolved
}

return (fnvPrime * bytes.Length * (((fnvPrime * (bytes[0] + 7)) ^ (bytes[^1] + 23)) + 11)) ^ (bytes[(bytes.Length - 1) / 2] + 53);
return (int)crc;
}

public static void ChangeEndianness8(Span<byte> bytes)
Expand Down