Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fast Keccak cache (wait-free and lockless) #7336

Merged
merged 53 commits into from
Sep 19, 2024
Merged
Changes from 3 commits
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
280097e
More comments
Scooletz Aug 16, 2024
7ecd7a5
comments
Scooletz Aug 16, 2024
8dce0c7
updated
Scooletz Aug 16, 2024
dec33ca
EVM uses the cached keccak
Scooletz Aug 16, 2024
f5b756c
state tree
Scooletz Aug 16, 2024
df510fd
more tests
Scooletz Aug 16, 2024
154037e
special cases
Scooletz Aug 16, 2024
24a595b
Release with Volatile.Write
Scooletz Aug 16, 2024
790f86d
StorageTree added
Scooletz Aug 16, 2024
2593734
comments
Scooletz Aug 16, 2024
805e28c
one less CAS
Scooletz Aug 16, 2024
b01d503
more go-tos
Scooletz Aug 16, 2024
83e8489
Merge branch 'master' into keccak-cache
benaadams Aug 16, 2024
0118c1d
one less branch in hash
Scooletz Aug 16, 2024
af55085
Merge branch 'keccak-cache' of https://github.com/NethermindEth/nethe…
Scooletz Aug 16, 2024
21d7714
Merge branch 'master' into keccak-cache
benaadams Aug 16, 2024
2e74b76
Start with per instance random
benaadams Aug 16, 2024
a29dcec
Include length in hash seed
benaadams Aug 16, 2024
e654d13
Improve comment
benaadams Aug 16, 2024
c88d931
Move stackalloc out of common path and inline ComputeKey to it
benaadams Aug 17, 2024
6fcfb9a
Unify hashing
benaadams Aug 17, 2024
381511e
Use full hash
benaadams Aug 18, 2024
9e2ea27
Faster StorageCell equality
benaadams Aug 18, 2024
c3996a2
Less copy
benaadams Aug 18, 2024
669cec4
Merge branch 'master' into keccak-cache
benaadams Aug 18, 2024
d1483ce
Even less copy
benaadams Aug 18, 2024
2070cc2
Doesn't need to return
benaadams Aug 19, 2024
28e0fa4
Update alignment comments
benaadams Aug 19, 2024
89bd655
lol; don't do extra work
benaadams Aug 19, 2024
42ef8f6
Faster compares
benaadams Aug 19, 2024
773ef2a
Move other HasCodes to FastHash
benaadams Aug 19, 2024
226be3f
Special case 32 and 20 bytes hashes
benaadams Aug 19, 2024
7d5fba5
Word align vector compare
benaadams Aug 19, 2024
7184d26
constants
Scooletz Aug 19, 2024
795b4c9
stack reduced by 8 and one less comparison on read
Scooletz Aug 19, 2024
c19edb3
Use full entropy of HashCode for comparision
benaadams Aug 19, 2024
1f8b94f
Revert "Use full entropy of HashCode for comparision"
benaadams Aug 19, 2024
928929e
Tweaks
benaadams Aug 19, 2024
e32f995
comments
Scooletz Aug 19, 2024
6938aab
Align 32
benaadams Aug 19, 2024
545384f
Merge branch 'keccak-cache' of https://github.com/NethermindEth/nethe…
benaadams Aug 19, 2024
472ca26
smaller entry, bigger cache
Scooletz Aug 20, 2024
7d1bba6
Missed one
benaadams Aug 20, 2024
b8a6f7f
Merge branch 'master' into keccak-cache
benaadams Aug 20, 2024
9e8b635
Merge branch 'master' into keccak-cache
benaadams Aug 23, 2024
a536245
Merge branch 'master' into keccak-cache
benaadams Aug 31, 2024
50c1e8f
Faster FastHash
benaadams Aug 31, 2024
fc52fcc
alignment handled with if
Scooletz Sep 2, 2024
c11cc94
Merge branch 'master' into keccak-cache
benaadams Sep 11, 2024
ce44b4c
Merge branch 'master' into keccak-cache
benaadams Sep 11, 2024
a2a0155
Merge branch 'master' into keccak-cache
benaadams Sep 19, 2024
7363f2c
Missed one fastHash
benaadams Sep 19, 2024
70f5995
Add memory pressure
benaadams Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 15 additions & 17 deletions src/Nethermind/Nethermind.Core/Crypto/KeccakCache.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,15 +67,14 @@ public static void ComputeTo(ReadOnlySpan<byte> input, out ValueHash256 keccak25
Debug.Assert(index < Count);

ref Entry e = ref Unsafe.Add(ref Unsafe.AsRef<Entry>(Memory), index);

// Half the hash his encoded in the bucket so we only need half of it and can use other half for length.
uint combined = (HashMask & (uint)hashCode) | (uint)input.Length;

// Check combined. It's aligned properly and can be read in one go.
uint existing = Volatile.Read(ref e.Combined);

// Lock by negating length if the length is the same size as input.
if (existing == combined &&
Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing)
if (Volatile.Read(ref e.Combined) == combined &&
Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, combined) == combined)
{
// Combined is equal to existing, meaning that it was not locked, and both the length and the hash were equal.

Expand Down Expand Up @@ -104,10 +103,10 @@ public static void ComputeTo(ReadOnlySpan<byte> input, out ValueHash256 keccak25
// Hashing Address
ref byte bytes0 = ref copy.Start;
ref byte bytes1 = ref MemoryMarshal.GetReference(input);
// 20 bytes which is Vector128+uint
if (Unsafe.As<byte, Vector128<byte>>(ref bytes0) == Unsafe.As<byte, Vector128<byte>>(ref bytes1) &&
Unsafe.As<byte, uint>(ref Unsafe.Add(ref bytes0, Vector128<byte>.Count)) ==
Unsafe.As<byte, uint>(ref Unsafe.Add(ref bytes1, Vector128<byte>.Count)))
// 20 bytes which is uint+Vector128
if (Unsafe.As<byte, uint>(ref bytes0) == Unsafe.As<byte, uint>(ref bytes1) &&
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref bytes0, sizeof(uint))) ==
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref bytes1, sizeof(uint))))
{
// Current keccak256 is correct hash.
return;
Expand All @@ -124,10 +123,9 @@ public static void ComputeTo(ReadOnlySpan<byte> input, out ValueHash256 keccak25
keccak256 = ValueKeccak.Compute(input);

// Try lock and memoize
existing = Volatile.Read(ref e.Combined);
uint existing = Volatile.Read(ref e.Combined);

// Negative value means that the entry is locked, set to int.MinValue to avoid confusion empty entry,
// since we are overwriting it anyway e.g. -0 would not be a reliable locked state.
// Set to locked to combined locked state, if not already locked.
if ((existing & LockMarker) == 0 && Interlocked.CompareExchange(ref e.Combined, combined | LockMarker, existing) == existing)
{
e.Keccak256 = keccak256;
Expand All @@ -144,10 +142,10 @@ public static void ComputeTo(ReadOnlySpan<byte> input, out ValueHash256 keccak25
// Address
ref byte bytes0 = ref e.Value.Start;
ref byte bytes1 = ref MemoryMarshal.GetReference(input);
// 20 bytes which is Vector128+uint
Unsafe.As<byte, Vector128<byte>>(ref bytes0) = Unsafe.As<byte, Vector128<byte>>(ref bytes1);
Unsafe.As<byte, uint>(ref Unsafe.Add(ref bytes0, Vector128<byte>.Count)) =
Unsafe.As<byte, uint>(ref Unsafe.Add(ref bytes1, Vector128<byte>.Count));
// 20 bytes which is uint+Vector128
Unsafe.As<byte, uint>(ref bytes0) = Unsafe.As<byte, uint>(ref bytes1);
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref bytes0, sizeof(uint))) =
Unsafe.As<byte, Vector128<byte>>(ref Unsafe.Add(ref bytes1, sizeof(uint)));
}
else
{
Expand All @@ -161,7 +159,7 @@ public static void ComputeTo(ReadOnlySpan<byte> input, out ValueHash256 keccak25

return;

Uncommon:
Uncommon:
if (input.Length == 0)
{
keccak256 = ValueKeccak.OfAnEmptyString;
Expand All @@ -188,7 +186,7 @@ private struct Entry
/// </summary>
public const int Size = 128;

private const int PayloadStart = 8;
private const int PayloadStart = sizeof(uint);
Scooletz marked this conversation as resolved.
Show resolved Hide resolved
private const int ValueStart = Size - ValueHash256.MemorySize;
public const int MaxPayloadLength = ValueStart - PayloadStart;

Expand Down