Skip to content

Commit

Permalink
Add JsonSelectSettings and regex timeout
Browse files Browse the repository at this point in the history
  • Loading branch information
@JamesNK
JamesNK committed Jan 17, 2021
1 parent 95a6eb3 commit 42139ea
Show file tree
Hide file tree
Showing 20 changed files with 108 additions and 193 deletions.
2 changes: 0 additions & 2 deletions Src/Directory.Build.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,5 @@
<SystemValueTuplePackageVersion>4.4.0</SystemValueTuplePackageVersion>
<XunitPackageVersion>2.3.1</XunitPackageVersion>
<XunitRunnerVisualStudioPackageVersion>2.3.1</XunitRunnerVisualStudioPackageVersion>
<BogusPackageVersion>32.0.2</BogusPackageVersion>
<AsyncExPackageVersion>5.1.0</AsyncExPackageVersion>
</PropertyGroup>
</Project>
68 changes: 8 additions & 60 deletions Src/Newtonsoft.Json.Tests/Linq/JsonPath/JPathExecuteTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,7 @@
using Newtonsoft.Json.Linq.JsonPath;
using Newtonsoft.Json.Tests.Bson;
#if HAVE_REGEX_TIMEOUTS
using Bogus;
using Nito.AsyncEx;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Threading;
#endif
#if DNXCORE50
using Xunit;
Expand Down Expand Up @@ -81,68 +77,20 @@ public void GreaterThanIssue1518()
[Test]
public void BacktrackingRegex_SingleMatch_TimeoutRespected()
{
var RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
const string RegexBacktrackingPattern = "(?<a>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";

var faker = new Faker();
var regexBacktrackingData = new JArray();
regexBacktrackingData.Add(new JObject(new JProperty("b", @"15/04/2020 8:18:03 PM|1|System.String[]|3|Libero eligendi magnam ut inventore.. Quaerat et sit voluptatibus repellendus blanditiis aliquam ut.. Quidem qui ut sint in ex et tempore.|||.\iste.cpp||46018|-1")));

for (var i = 0; i < 1000; i++)
ExceptionAssert.Throws<RegexMatchTimeoutException>(() =>
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}

Xunit.Assert.Throws<RegexMatchTimeoutException>(() =>
{
var tokens = regexBacktrackingData.SelectTokens(
regexBacktrackingData.SelectTokens(
$"[?(@.b =~ /{RegexBacktrackingPattern}/)]",
errorWhenNoMatch: false,
singleRegexMatchTimeout: TimeSpan.FromSeconds(.01)).ToArray();
});
}

[Test]
public void BacktrackingRegex_GlobalMatch_TimeoutRespected()
{
var faker = new Faker();
var regexBacktrackingData = new JArray();
for (var i = 0; i < 1000; i++)
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}
var RegexBacktrackingPattern = "(?<i>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
var jpathExpression = $"[?(@.b =~ /{RegexBacktrackingPattern}/c)]";

var exceptionThrow = Xunit.Assert.ThrowsAny<Exception>(() =>
{
var tokens = regexBacktrackingData.SelectTokens(
jpathExpression,
errorWhenNoMatch: false,
singleRegexMatchTimeout: TimeSpan.FromSeconds(2),
globalRegexMatchTimeout: TimeSpan.FromSeconds(.5)).ToArray();
new JsonSelectSettings
{
RegexMatchTimeout = TimeSpan.FromSeconds(0.01)
}).ToArray();
});
Assert.IsTrue(exceptionThrow is OperationCanceledException or RegexMatchTimeoutException);
}

[Test]
public async Task BacktrackingRegexCanBeVerySlowWithoutTimeouts()
{
var RegexBacktrackingPattern = "(?<j>(.*?))[|].*(?<b>(.*?))[|].*(?<c>(.*?))[|].*(?<d>[1-3])[|].*(?<e>(.*?))[|].*[|].*[|].*(?<f>(.*?))[|].*[|].*(?<g>(.*?))[|].*(?<h>(.*))";
var faker = new Faker();
var regexBacktrackingData = new JArray();

for (var i = 0; i < 10; i++)
{
var value = $"{faker.Date.Past()}|1|{faker.Lorem.Words()}|3|{faker.Lorem.Sentences(3, ". ")}|||.\\{faker.Lorem.Word()}.cpp||{faker.Random.UShort()}|-1";
regexBacktrackingData.Add(new JObject(new JProperty("b", value)));
}

var selectTokenTask = Task.Run(() => regexBacktrackingData.SelectTokens($"[?(@.b =~ /{RegexBacktrackingPattern}/)]").ToArray());
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(.3));
using var ctts = new CancellationTokenTaskSource<bool>(cts.Token);
var finishedTask = await Task.WhenAny(selectTokenTask, ctts.Task);
Assert.AreEqual(finishedTask, ctts.Task);
}
#endif

Expand Down
11 changes: 1 addition & 10 deletions Src/Newtonsoft.Json.Tests/Newtonsoft.Json.Tests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
<PropertyGroup Condition="'$(TargetFramework)'=='net46'">
<AssemblyTitle>Json.NET Tests</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETFramework,Version=v4.5</ReferringTargetFrameworkForProjectReferences>
<DefineConstants>NET45;HAVE_BENCHMARKS;$(AdditionalConstants)</DefineConstants>
<DefineConstants>NET45;HAVE_BENCHMARKS;HAVE_REGEX_TIMEOUTS;$(AdditionalConstants)</DefineConstants>
</PropertyGroup>

<ItemGroup Condition="'$(TargetFramework)'=='net40'">
Expand All @@ -76,7 +76,6 @@
<Reference Include="System.ComponentModel.DataAnnotations" />
<Reference Include="System.Web.Extensions" />
<Reference Include="System.Data.DataSetExtensions" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='net40'">
<AssemblyTitle>Json.NET Tests .NET 4.0</AssemblyTitle>
Expand Down Expand Up @@ -122,10 +121,7 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>

<PropertyGroup Condition="'$(TargetFramework)'=='net5.0'">
<AssemblyTitle>Json.NET Tests .NET Standard 2.0</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v2.0</ReferringTargetFrameworkForProjectReferences>
Expand All @@ -146,8 +142,6 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>
<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp3.1'">
<AssemblyTitle>Json.NET Tests .NET Standard 1.3</AssemblyTitle>
Expand All @@ -168,10 +162,7 @@
<PackageReference Include="xunit" Version="$(XunitPackageVersion)" />
<PackageReference Include="xunit.runner.visualstudio" Version="$(XunitRunnerVisualStudioPackageVersion)" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="$(MicrosoftNETTestSdkPackageVersion)" />
<PackageReference Include="Bogus" Version="$(BogusPackageVersion)" />
<PackageReference Include="Nito.AsyncEx.Tasks" Version="$(AsyncExPackageVersion)" />
</ItemGroup>

<PropertyGroup Condition="'$(TargetFramework)'=='netcoreapp2.1'">
<AssemblyTitle>Json.NET Tests .NET Standard 1.0</AssemblyTitle>
<ReferringTargetFrameworkForProjectReferences>.NETStandard,Version=v1.0</ReferringTargetFrameworkForProjectReferences>
Expand Down
75 changes: 19 additions & 56 deletions Src/Newtonsoft.Json/Linq/JToken.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@
using Newtonsoft.Json.Utilities.LinqBridge;
#else
using System.Linq;
using System.Threading;
#endif

namespace Newtonsoft.Json.Linq
Expand Down Expand Up @@ -2308,7 +2307,7 @@ int IJsonLineInfo.LinePosition
/// <returns>A <see cref="JToken"/>, or <c>null</c>.</returns>
public JToken? SelectToken(string path)
{
return SelectToken(path, false);
return SelectToken(path, settings: null);
}

/// <summary>
Expand All @@ -2321,38 +2320,27 @@ int IJsonLineInfo.LinePosition
/// <returns>A <see cref="JToken"/>.</returns>
public JToken? SelectToken(string path, bool errorWhenNoMatch)
{
JPath p = new JPath(path);
JsonSelectSettings? settings = errorWhenNoMatch
? new JsonSelectSettings { ErrorWhenNoMatch = true }
: null;

JToken? token = null;
foreach (JToken t in p.Evaluate(this, this, errorWhenNoMatch))
{
if (token != null)
{
throw new JsonException("Path returned multiple tokens.");
}

token = t;
}

return token;
return SelectToken(path, settings);
}

#if HAVE_REGEX_TIMEOUTS
/// <summary>
/// Selects a <see cref="JToken"/> using a JSONPath expression. Selects the token that matches the object path.
/// </summary>
/// <param name="path">
/// A <see cref="String"/> that contains a JSONPath expression.
/// </param>
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
/// <param name="singleMatchTimeout">the time after which a single call to regex.ismatch must complete, default is forever</param>
/// <param name="settings">The <see cref="JsonSelectSettings"/> used to select tokens.</param>
/// <returns>A <see cref="JToken"/>.</returns>
public JToken? SelectToken(string path, bool errorWhenNoMatch, TimeSpan? singleMatchTimeout = default)
public JToken? SelectToken(string path, JsonSelectSettings? settings)
{
JPath p = new JPath(path, singleMatchTimeout);
JPath p = new JPath(path);

JToken? token = null;
foreach (JToken t in p.Evaluate(this, this, errorWhenNoMatch))
foreach (JToken t in p.Evaluate(this, this, settings))
{
if (token != null)
{
Expand All @@ -2364,7 +2352,6 @@ int IJsonLineInfo.LinePosition

return token;
}
#endif

/// <summary>
/// Selects a collection of elements using a JSONPath expression.
Expand All @@ -2375,7 +2362,7 @@ int IJsonLineInfo.LinePosition
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
public IEnumerable<JToken> SelectTokens(string path)
{
return SelectTokens(path, false);
return SelectTokens(path, settings: null);
}

/// <summary>
Expand All @@ -2388,50 +2375,26 @@ public IEnumerable<JToken> SelectTokens(string path)
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
public IEnumerable<JToken> SelectTokens(string path, bool errorWhenNoMatch)
{
var p = new JPath(path);
return p.Evaluate(this, this, errorWhenNoMatch);
JsonSelectSettings? settings = errorWhenNoMatch
? new JsonSelectSettings { ErrorWhenNoMatch = true }
: null;

return SelectTokens(path, settings);
}

#if HAVE_REGEX_TIMEOUTS
/// <summary>
/// Selects a collection of elements using a JSONPath expression.
/// </summary>
/// <param name="path">
/// A <see cref="String"/> that contains a JSONPath expression.
/// </param>
/// <param name="errorWhenNoMatch">A flag to indicate whether an error should be thrown if no tokens are found when evaluating part of the expression.</param>
/// <param name="singleRegexMatchTimeout">
/// for every token that matches a jpath, the time a regex is permitted to run
/// against that token before timeout
/// </param>
/// <param name="globalRegexMatchTimeout">
/// the time this method should wait for the given jpath regex to match all tokens.
/// worst case expected execution time roughly globalRegexMatchTimeout + Min(singleRegexMatchTimeout, globalRegexMatchTimeout)
/// </param>
/// <exception cref="System.Text.RegularExpressions.RegexMatchTimeoutException">if single call timeout exceeded</exception>"
/// <param name="settings">The <see cref="JsonSelectSettings"/> used to select tokens.</param>
/// <returns>An <see cref="IEnumerable{T}"/> of <see cref="JToken"/> that contains the selected elements.</returns>
public IEnumerable<JToken> SelectTokens(string path,
bool errorWhenNoMatch,
TimeSpan? singleRegexMatchTimeout = default,
TimeSpan? globalRegexMatchTimeout = default)
public IEnumerable<JToken> SelectTokens(string path, JsonSelectSettings? settings)
{
var singleTimeout = singleRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
var globalTimeout = globalRegexMatchTimeout ?? Timeout.InfiniteTimeSpan;
if (globalTimeout != Timeout.InfiniteTimeSpan && globalTimeout < singleTimeout)
{
singleTimeout = globalTimeout;
}

using var cts = new CancellationTokenSource(globalTimeout);
var p = new JPath(path, singleTimeout);
var results = p.Evaluate(this, this, errorWhenNoMatch);
foreach (var result in results)
{
cts.Token.ThrowIfCancellationRequested();
yield return result;
}
var p = new JPath(path);
return p.Evaluate(this, this, settings);
}
#endif

#if HAVE_DYNAMIC
/// <summary>
Expand Down
6 changes: 3 additions & 3 deletions Src/Newtonsoft.Json/Linq/JsonPath/ArrayIndexFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,13 @@ internal class ArrayIndexFilter : PathFilter
{
public int? Index { get; set; }

public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
{
foreach (JToken t in current)
{
if (Index != null)
{
JToken? v = GetTokenIndex(t, errorWhenNoMatch, Index.GetValueOrDefault());
JToken? v = GetTokenIndex(t, settings, Index.GetValueOrDefault());

if (v != null)
{
Expand All @@ -32,7 +32,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
}
else
{
if (errorWhenNoMatch)
if (settings?.ErrorWhenNoMatch ?? false)
{
throw new JsonException("Index * not valid on {0}.".FormatWith(CultureInfo.InvariantCulture, t.GetType().Name));
}
Expand Down
4 changes: 2 additions & 2 deletions Src/Newtonsoft.Json/Linq/JsonPath/ArrayMultipleIndexFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,13 @@ public ArrayMultipleIndexFilter(List<int> indexes)
Indexes = indexes;
}

public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
{
foreach (JToken t in current)
{
foreach (int i in Indexes)
{
JToken? v = GetTokenIndex(t, errorWhenNoMatch, i);
JToken? v = GetTokenIndex(t, settings, i);

if (v != null)
{
Expand Down
6 changes: 3 additions & 3 deletions Src/Newtonsoft.Json/Linq/JsonPath/ArraySliceFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ internal class ArraySliceFilter : PathFilter
public int? End { get; set; }
public int? Step { get; set; }

public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
{
if (Step == 0)
{
Expand Down Expand Up @@ -56,7 +56,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
}
else
{
if (errorWhenNoMatch)
if (settings?.ErrorWhenNoMatch ?? false)
{
throw new JsonException("Array slice of {0} to {1} returned no results.".FormatWith(CultureInfo.InvariantCulture,
Start != null ? Start.GetValueOrDefault().ToString(CultureInfo.InvariantCulture) : "*",
Expand All @@ -66,7 +66,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
}
else
{
if (errorWhenNoMatch)
if (settings?.ErrorWhenNoMatch ?? false)
{
throw new JsonException("Array slice is not valid on {0}.".FormatWith(CultureInfo.InvariantCulture, t.GetType().Name));
}
Expand Down
6 changes: 3 additions & 3 deletions Src/Newtonsoft.Json/Linq/JsonPath/FieldFilter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ public FieldFilter(string? name)
Name = name;
}

public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, bool errorWhenNoMatch)
public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToken> current, JsonSelectSettings? settings)
{
foreach (JToken t in current)
{
Expand All @@ -27,7 +27,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
{
yield return v;
}
else if (errorWhenNoMatch)
else if (settings?.ErrorWhenNoMatch ?? false)
{
throw new JsonException("Property '{0}' does not exist on JObject.".FormatWith(CultureInfo.InvariantCulture, Name));
}
Expand All @@ -42,7 +42,7 @@ public override IEnumerable<JToken> ExecuteFilter(JToken root, IEnumerable<JToke
}
else
{
if (errorWhenNoMatch)
if (settings?.ErrorWhenNoMatch ?? false)
{
throw new JsonException("Property '{0}' not valid on {1}.".FormatWith(CultureInfo.InvariantCulture, Name ?? "*", t.GetType().Name));
}
Expand Down
Loading

0 comments on commit 42139ea

Please sign in to comment.