Skip to content

Commit

Permalink
Improved Json Serialization (#6152)
Browse files Browse the repository at this point in the history
* System.Text.Json.Serialization

* System.Text.Deserialization

* Fixes

* lint

* Fixes

* Fix

* Fixes

* Fixes

* Lint

* Fix invalid Json

* u8

* Fixes

* Fixes

* Fix

* Don't go via exception for short reverts

* Fix

* Fix

* Multi-doc batch

* Lint

* Merge conflict

* Fixes

* Lint

* Fix tests

* Fix test

* Tidy up

* Turn of sync IO

* Async

* Don't double Advance

* Async wrapper stream use

* Fix build break

* Merge issues

* Compile issues

* File scoped namespaces

* Formatting

* Merge fixes

* Test Merge fixes

* lang ver

* Purge Newtonsoft

* Fix evm tests

* Fix parentBeaconBlockRootString serialization

* Change type for serialization

* Ignore syncing

* Capitalised bools are invalid json

* Capitalised bools are invalid json

* Fix merge conflict

* Fix data serialization

* Fix trace entry

* Merge conflicts

* Formatting

* Missed merge

* whitespace

* Fix EthereumTests Json

* Ethereum tests

* Fix hex prefix

* Fix difficultly test

* Add serializer options

* Fix abi tests

* Merge conflict

* Feedback

* revert to previous

* Remove Netwonsoft

* cosmetics

* Pipeline Json responses

* Merge fixes

* Whitespace

* fix test

* Send multiple messages

* Optional stream closing

* Case insensitive

* Update Nethermind.Serialization.Json.csproj

* Whitespace

* Update JavaScriptObjectConverter.cs

* Feedback

* Undo AI

* Put orders back

* Feedback

* Whitespace

* Use static constructor

* Remove irrelevant test parameter

* Add BigIntegerConverterTests back

* Use FrozenDictionary

* Deserialize collection to ArrayPoolList using GetArrayLength for initial capacity. Dispose it later.

Move WhiteSpace to utf8

* simplify

* Throw more explicit exception

* name cleanup

* Merge conflicts

* Merge conflict

* Add comments

* more comments

* Consolidate Disposable

* Use FrozenDictionary for RpcLookup

* Case sensitive

* Be consistent on parsing type

* Use correct type in test

* Optimizations

* Optimizations

* Optimize

* Optimize

* Vectorize FromUtf8 hex

* Optimization

* Optimize

* Parallel deserialize

* Revert "Parallel deserialize"

This reverts commit 94b08e3.

* Use using

* Fix check

* Parallel deserialize

* Increase parallelThreshold

* Use correct length

* Cache Reflection Parameters

* Fix tracing output

* Match geth output

* Turn AllowSynchronousIO back on

* Revert object type arrays

---------

Co-authored-by: lukasz.rozmej <lukasz.rozmej@gmail.com>
  • Loading branch information
benaadams and LukaszRozmej authored Dec 11, 2023
1 parent 3f2f149 commit 82b816a
Show file tree
Hide file tree
Showing 289 changed files with 8,510 additions and 6,291 deletions.
2 changes: 1 addition & 1 deletion src/Nethermind/Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
<PackageVersion Include="Nethermind.DotNetty.Transport" Version="1.0.1" />
<PackageVersion Include="Nethermind.Gmp" Version="1.0.1" />
<PackageVersion Include="Nethermind.Numerics.Int256" Version="1.1.1" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="Nito.Collections.Deque" Version="1.2.1" />
<PackageVersion Include="NLog" Version="5.2.7" />
<PackageVersion Include="NLog.Targets.Seq" Version="3.0.0" />
Expand All @@ -79,6 +78,7 @@
<PackageVersion Include="System.Diagnostics.TextWriterTraceListener" Version="4.3.0" />
<PackageVersion Include="System.IdentityModel.Tokens.Jwt" Version="7.0.3" />
<PackageVersion Include="System.IO.Compression.ZipFile" Version="4.3.0" />
<PackageVersion Include="System.IO.Pipelines" Version="8.0.0" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Net.Http" Version="4.3.4" />
<PackageVersion Include="System.Runtime.Caching" Version="8.0.0" />
Expand Down
8 changes: 4 additions & 4 deletions src/Nethermind/Ethereum.Abi.Test/AbiTest.cs
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using Newtonsoft.Json;
using System.Text.Json.Serialization;

namespace Ethereum.Abi.Test
{
public class AbiTest
{
[JsonProperty(PropertyName = "args")]
[JsonPropertyName("args")]
public object[] Args { get; set; }

[JsonProperty(PropertyName = "result")]
[JsonPropertyName("result")]
public string Result { get; set; }

[JsonProperty(PropertyName = "types")]
[JsonPropertyName("types")]
public string[] Types { get; set; }
}
}
11 changes: 6 additions & 5 deletions src/Nethermind/Ethereum.Abi.Test/Tests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Nodes;

using FluentAssertions;
using Nethermind.Abi;
using Nethermind.Core.Extensions;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NUnit.Framework;

namespace Ethereum.Abi.Test
Expand Down Expand Up @@ -56,7 +57,7 @@ public void Test_abi_encoding()
}
}

Dictionary<string, AbiTest> tests = JsonConvert.DeserializeObject<Dictionary<string, AbiTest>>(text);
Dictionary<string, AbiTest> tests = JsonSerializer.Deserialize<Dictionary<string, AbiTest>>(text);
foreach ((string testName, AbiTest abiTest) in tests)
{
AbiSignature signature = new(
Expand All @@ -71,9 +72,9 @@ public void Test_abi_encoding()

public object JsonToObject(object jsonObject)
{
if (jsonObject is JArray array)
if (jsonObject is JsonArray array)
{
return array.Select(t => t.Value<long>()).ToArray();
return array.Select(t => t.GetValue<long>()).ToArray();
}

return jsonObject;
Expand Down
2 changes: 2 additions & 0 deletions src/Nethermind/Ethereum.Basic.Test/TransactionTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
using System.IO;
using System.Linq;
using System.Numerics;
using System.Text.Json.Serialization;

using Ethereum.Test.Base;
using Nethermind.Core;
using Nethermind.Core.Extensions;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" />
<PackageReference Include="NUnit" />
<PackageReference Include="NUnit.Analyzers">
<PrivateAssets>all</PrivateAssets>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Newtonsoft.Json" />
<PackageReference Include="NUnit3TestAdapter" />
</ItemGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ public class TransactionJsonTest : GeneralStateTestBase
public void Can_load_access_lists()
{
const string lists =
"{\"accessLists\": [[{address: \"0x0001020304050607080900010203040506070809\", storageKeys: [\"0x00\", \"0x01\"]}]]}";
"{\"accessLists\": [[{\"address\": \"0x0001020304050607080900010203040506070809\", \"storageKeys\": [\"0x00\", \"0x01\"]}]]}";

EthereumJsonSerializer serializer = new EthereumJsonSerializer();
TransactionJson txJson = serializer.Deserialize<TransactionJson>(lists);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System.Text.Json.Serialization;

namespace Ethereum.Difficulty.Test
{
public class DifficultyTestHexJson
Expand Down
2 changes: 2 additions & 0 deletions src/Nethermind/Ethereum.Difficulty.Test/DifficultyTestJson.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System.Text.Json.Serialization;

namespace Ethereum.Difficulty.Test
{
public class DifficultyTestJson
Expand Down
7 changes: 5 additions & 2 deletions src/Nethermind/Ethereum.HexPrefix.Test/HexPrefixTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@

using System.Collections.Generic;
using System.Linq;
using System.Text.Json.Serialization;

using Ethereum.Test.Base;
using Nethermind.Core.Extensions;

using NUnit.Framework;

namespace Ethereum.HexPrefix.Test
Expand All @@ -18,7 +21,7 @@ public static IEnumerable<HexPrefixTest> LoadTests()
{
return TestLoader.LoadFromFile<Dictionary<string, HexPrefixTestJson>, HexPrefixTest>(
"hexencodetest.json",
c => c.Select(p => new HexPrefixTest(p.Key, p.Value.Seq, p.Value.Term, p.Value.Out)));
c => c.Select(p => new HexPrefixTest(p.Key, p.Value.Seq.Select(x => (byte)x).ToArray(), p.Value.Term, p.Value.Out)));
}

[TestCaseSource(nameof(LoadTests))]
Expand All @@ -36,7 +39,7 @@ public void Test(HexPrefixTest test)

private class HexPrefixTestJson
{
public byte[] Seq { get; set; }
public int[] Seq { get; set; }
public bool Term { get; set; }
public string Out { get; set; }
}
Expand Down
5 changes: 3 additions & 2 deletions src/Nethermind/Ethereum.KeyAddress.Test/KeyAddressTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json.Serialization;

using Ethereum.Test.Base;
using Nethermind.Core;
using Nethermind.Core.Crypto;
using Nethermind.Crypto;
using Nethermind.Int256;
using Nethermind.Logging;
using Newtonsoft.Json;
using NUnit.Framework;

namespace Ethereum.KeyAddress.Test
Expand Down Expand Up @@ -96,7 +97,7 @@ private class SigOfEmptyString

private class KeyAddressTestJson
{
[JsonProperty("sig_of_emptystring")]
[JsonPropertyName("sig_of_emptystring")]
public SigOfEmptyString Signature { get; set; }

public string Seed { get; set; }
Expand Down
6 changes: 4 additions & 2 deletions src/Nethermind/Ethereum.KeyStore.Test/KeyStoreJsonTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
using System;
using System.IO;
using System.Security;
using System.Text.Json.Serialization;

using Nethermind.Core;
using Nethermind.Core.Collections;
using Nethermind.Crypto;
Expand All @@ -12,7 +14,7 @@
using Nethermind.KeyStore.Config;
using Nethermind.Logging;
using Nethermind.Serialization.Json;
using Newtonsoft.Json;

using NUnit.Framework;

namespace Ethereum.KeyStore.Test
Expand Down Expand Up @@ -129,7 +131,7 @@ private class KeyStoreTestsModel

private class KeyStoreTestModel
{
[JsonProperty(PropertyName = "Json")]
[JsonPropertyName("Json")]
public KeyStoreItem KeyData { get; set; }
public string Password { get; set; }
public string Priv { get; set; }
Expand Down
11 changes: 6 additions & 5 deletions src/Nethermind/Ethereum.PoW.Test/EthashTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
using System.IO;
using System.Linq;
using System.Numerics;
using System.Text.Json.Serialization;

using Ethereum.Test.Base;
using Nethermind.Consensus.Ethash;
using Nethermind.Core;
Expand All @@ -15,7 +17,6 @@
using Nethermind.Crypto;
using Nethermind.Logging;
using Nethermind.Serialization.Rlp;
using Newtonsoft.Json;
using NUnit.Framework;

namespace Ethereum.PoW.Test
Expand Down Expand Up @@ -103,16 +104,16 @@ private class EthashTestJson
public string Header { get; set; }
public string Seed { get; set; }

[JsonProperty("cache_size")]
[JsonPropertyName("cache_size")]
public int CacheSize { get; set; }

[JsonProperty("full_size")]
[JsonPropertyName("full_size")]
public int FullSize { get; set; }

[JsonProperty("header_hash")]
[JsonPropertyName("header_hash")]
public string HeaderHash { get; set; }

[JsonProperty("cache_hash")]
[JsonPropertyName("cache_hash")]
public string CacheHash { get; set; }

public string Result { get; set; }
Expand Down
3 changes: 0 additions & 3 deletions src/Nethermind/Ethereum.Test.Base/AccessListJson.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,13 @@
// SPDX-License-Identifier: LGPL-3.0-only

using Nethermind.Core;
using Newtonsoft.Json;

namespace Ethereum.Test.Base
{
public class AccessListItemJson
{
[JsonProperty("address")]
public Address Address { get; set; }

[JsonProperty("storageKeys")]
public byte[][] StorageKeys { get; set; }
}
}
3 changes: 2 additions & 1 deletion src/Nethermind/Ethereum.Test.Base/EthereumTestResult.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using System.Text.Json.Serialization;

using Nethermind.Core.Crypto;
using Newtonsoft.Json;

namespace Ethereum.Test.Base
{
Expand Down
4 changes: 2 additions & 2 deletions src/Nethermind/Ethereum.Test.Base/GeneralStateTestJson.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@
// SPDX-License-Identifier: LGPL-3.0-only

using System.Collections.Generic;
using Newtonsoft.Json;
using System.Text.Json.Serialization;

namespace Ethereum.Test.Base
{
public class GeneralStateTestJson
{
[JsonProperty("_info")]
[JsonPropertyName("_info")]
public GeneralStateTestInfoJson? Info { get; set; }
public GeneralStateTestEnvJson? Env { get; set; }
public Dictionary<string, PostStateJson[]>? Post { get; set; }
Expand Down
5 changes: 3 additions & 2 deletions src/Nethermind/Ethereum.Test.Base/TestBlockJson.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
// SPDX-License-Identifier: LGPL-3.0-only

using Newtonsoft.Json;
using System.Text.Json;
using System.Text.Json.Serialization;

namespace Ethereum.Test.Base
{
Expand All @@ -11,7 +12,7 @@ public class TestBlockJson
public TestBlockHeaderJson[]? UncleHeaders { get; set; }
public string? Rlp { get; set; }
public LegacyTransactionJson[]? Transactions { get; set; }
[JsonProperty("expectException")]
[JsonPropertyName("expectException")]
public string? ExpectedException { get; set; }
}
}
46 changes: 29 additions & 17 deletions src/Nethermind/Ethereum.Test.Base/TestLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,14 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Numerics;
using System.Reflection;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;

using NUnit.Framework;

namespace Ethereum.Test.Base
Expand All @@ -17,30 +19,33 @@ public static class TestLoader
{
public static object PrepareInput(object input)
{
string s = input as string;
if (s != null && s.StartsWith("#"))
if (input is string s && s.StartsWith("#"))
{
BigInteger bigInteger = BigInteger.Parse(s.Substring(1));
input = bigInteger;
}

if (input is JArray)
{
input = ((JArray)input).Select(PrepareInput).ToArray();
}
JsonElement token = (JsonElement)input;

JToken token = input as JToken;
if (token != null)
if (token.ValueKind == JsonValueKind.Array)
{
if (token.Type == JTokenType.String)
object[] array = new object[token.GetArrayLength()];
for (int i = 0; i < array.Length; i++)
{
return token.Value<string>();
array[i] = PrepareInput(token[i]);
}

if (token.Type == JTokenType.Integer)
{
return token.Value<long>();
}
input = array;
}

if (token.ValueKind == JsonValueKind.String)
{
return token.GetString()!;
}

if (token.ValueKind == JsonValueKind.Number)
{
return token.GetInt64();
}

return input;
Expand All @@ -58,14 +63,21 @@ public static IEnumerable<TTest> LoadFromFile<TContainer, TTest>(
throw new ArgumentException($"Cannot find test resource: {testFileName}");
}

var jsonOptions = new JsonSerializerOptions()
{
PropertyNameCaseInsensitive = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DictionaryKeyPolicy = JsonNamingPolicy.CamelCase,
NumberHandling = JsonNumberHandling.AllowReadingFromString
};
using (Stream stream = assembly.GetManifestResourceStream(resourceName))
{
Assert.NotNull(stream);
using (StreamReader reader = new(stream))
{
string testJson = reader.ReadToEnd();
TContainer testSpecs =
JsonConvert.DeserializeObject<TContainer>(testJson);
JsonSerializer.Deserialize<TContainer>(testJson, jsonOptions);
return testExtractor(testSpecs);
}
}
Expand Down
Loading

0 comments on commit 82b816a

Please sign in to comment.