Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[HTTP/3] Stress hack for msquic dropping connections #84793

Merged
merged 13 commits into from
Apr 28, 2023
Merged

[HTTP/3] Stress hack for msquic dropping connections #84793

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
df45d59
Implement the same hack as for functional tests to prevent msquic fro…
ManickaP Apr 13, 2023
df7b541
Feedback: removed code sharing and used reflaction
ManickaP Apr 14, 2023
bd6c04d
Try to fix missing build dependency
ManickaP Apr 14, 2023
fa5820e
Feedback - removed test only function and replaced with shared code +…
ManickaP Apr 20, 2023
b649716
Merge branch 'main' into mapichov/h3-stress-msquic-hack
antonfirsov Apr 25, 2023
ed73b02
fix argument handling in build-local.ps1
antonfirsov Apr 25, 2023
7e4d512
do not use MsQuicLibraryVersion
antonfirsov Apr 25, 2023
1f8f56e
copy msquic interop utils to the SDK base image
antonfirsov Apr 26, 2023
d2c113c
use LinkBase in Functional.Tests.csproj for wildcard include
antonfirsov Apr 26, 2023
7613f06
Merge branch 'main' into mapichov/h3-stress-msquic-hack
antonfirsov Apr 26, 2023
0a9731f
Use MsQuicLibraryVersion in out internal logging and as a side-effect…
ManickaP Apr 27, 2023
e316a13
Comment
ManickaP Apr 27, 2023
949d82b
also copy files to the Linux container
antonfirsov Apr 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions eng/docker/build-docker-sdk.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ if ($buildWindowsContainers)
# 2. Runtime pack (microsoft.netcore.app.runtime.win-x64)
# 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
# 4. testhost
# 5. msquic interop sources (needed for HttpStress)
$binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
$dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context"

Expand All @@ -51,6 +52,8 @@ if ($buildWindowsContainers)
-Destination $dockerContext\testhost
Copy-Item -Recurse -Path $REPO_ROOT_DIR\eng\targetingpacks.targets `
-Destination $dockerContext\targetingpacks.targets
Copy-Item -Recurse -Path $REPO_ROOT_DIR\src\libraries\System.Net.Quic\src\System\Net\Quic\Interop `
-Destination $dockerContext\msquic-interop

# In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
$testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
Expand Down
5 changes: 3 additions & 2 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@

<!-- Stress projects have their own global.json, the directory above that also has it is the repository root. -->
<RepositoryRoot>$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)../, global.json))/</RepositoryRoot>


<MsQuicInteropIncludes Condition="'$(MsQuicInteropIncludes)' == ''">$(RepositoryRoot)src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/*.cs</MsQuicInteropIncludes>
<TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">$(RepositoryRoot)eng/targetingpacks.targets</TargetingPacksTargetsLocation>
<ProductVersion>8.0.0</ProductVersion>
<NetCoreAppCurrent>net8.0</NetCoreAppCurrent>
Expand All @@ -14,4 +15,4 @@
<MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
<MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.runtime.$(OutputRID)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
</PropertyGroup>
</Project>
</Project>
7 changes: 7 additions & 0 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
<TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
<Nullable>enable</Nullable>
<EnablePreviewFeatures>True</EnablePreviewFeatures>
<NoWarn>CA2252</NoWarn>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<ItemGroup>
Expand All @@ -15,6 +17,11 @@
<PackageReference Include="System.Net.Http.WinHttpHandler" Version="4.5.4" />
</ItemGroup>

<!-- Shared production code -->
<ItemGroup>
<Compile Include="$(MsQuicInteropIncludes)" LinkBase="MsQuicInterop" />
</ItemGroup>

<PropertyGroup>
<!-- These may lead to duplicate generated classes with local (non-docker) Linux builds. -->
<GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
Expand Down
24 changes: 24 additions & 0 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
using System.Threading.Tasks;
using System.Net;
using HttpStress;
using System.Net.Quic;
using Microsoft.Quic;

[assembly:SupportedOSPlatform("windows")]
[assembly:SupportedOSPlatform("linux")]
Expand All @@ -26,6 +28,8 @@ public static class Program
{
public enum ExitCode { Success = 0, StressError = 1, CliError = 2 };

public static readonly bool IsQuicSupported = QuicListener.IsSupported && QuicConnection.IsSupported;

public static async Task<int> Main(string[] args)
{
if (!TryParseCli(args, out Configuration? config))
Expand Down Expand Up @@ -158,6 +162,9 @@ private static async Task<ExitCode> Run(Configuration config)

string GetAssemblyInfo(Assembly assembly) => $"{assembly.Location}, modified {new FileInfo(assembly.Location).LastWriteTime}";

Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");
string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ManickaP unfortunately, this line fails with NullReferenceException in release builds. I assume the MsQuicLibraryVersion is being trimmed away, since it's unused in the production code.

We could workaround this by moving the version extraction to a shared test-utility (together with the MaxWorkerQueue hack helper), but IMHO it's not worth it unless we think that it's super-important to know the msquic version for stress test troubleshooting.

For now I'm just removing it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it possible that there is image without MsQuic?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What do you mean? If you are suspecting lack of msquic as the root cause of this error - it's not the case. Removing code messing with MsQuicLibraryVersion makes things work (= HTTP/3 stress runs).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is it possible that there is image without MsQuic?

In that case you'd get "unknown" as the MsQuic version, the code expects that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another option would be to tell the trimmer to keep it there via DynamicDependency. And this is just one string.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should try to clarify whether test-only usage of DynamicDependency is ok, without that, I prefer to defensively assume it's not.


Console.WriteLine(" .NET Core: " + GetAssemblyInfo(typeof(object).Assembly));
Console.WriteLine(" ASP.NET Core: " + GetAssemblyInfo(typeof(WebHost).Assembly));
Console.WriteLine(" System.Net.Http: " + GetAssemblyInfo(typeof(System.Net.Http.HttpClient).Assembly));
Expand All @@ -169,6 +176,8 @@ private static async Task<ExitCode> Run(Configuration config)
Console.WriteLine(" Concurrency: " + config.ConcurrentRequests);
Console.WriteLine(" Content Length: " + config.MaxContentLength);
Console.WriteLine(" HTTP Version: " + config.HttpVersion);
Console.WriteLine(" QUIC supported: " + (IsQuicSupported ? "yes" : "no"));
Console.WriteLine(" MsQuic Version: " + msQuicLibraryVersion);
Console.WriteLine(" Lifetime: " + (config.ConnectionLifetime.HasValue ? $"{config.ConnectionLifetime.Value.TotalMilliseconds}ms" : "(infinite)"));
Console.WriteLine(" Operations: " + string.Join(", ", usedClientOperations.Select(o => o.name)));
Console.WriteLine(" Random Seed: " + config.RandomSeed);
Expand All @@ -177,6 +186,21 @@ private static async Task<ExitCode> Run(Configuration config)
Console.WriteLine("Query Parameters: " + config.MaxParameters);
Console.WriteLine();

if (config.HttpVersion == HttpVersion.Version30 && IsQuicSupported)
{
unsafe
{
object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
QUIC_SETTINGS settings = default(QUIC_SETTINGS);
settings.IsSet.MaxWorkerQueueDelayUs = 1;
settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
{
Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
}
}
}

StressServer? server = null;
if (config.RunMode.HasFlag(RunMode.server))
Expand Down
6 changes: 3 additions & 3 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/build-local.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ if (-not ([string]::IsNullOrEmpty($args[0]))) {

$LibrariesConfiguration = "Release"
if (-not ([string]::IsNullOrEmpty($args[1]))) {
$LibrariesConfiguration = $args[0]
$LibrariesConfiguration = $args[1]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixes in this file are unrelated. Both $StressConfiguration and $LibrariesConfiguration referred to $args[0] 🤦 .

}

$TestHostRoot="$RepoRoot/artifacts/bin/testhost/net$Version-windows-$LibrariesConfiguration-x64"
Expand Down Expand Up @@ -53,11 +53,11 @@ if (-not (Test-Path -Path "$TestHostRoot/shared/Microsoft.AspNetCore.App")) {
Write-Host "Building solution."
dotnet build -c $StressConfiguration

$Runscript=".\run-stress-$LibrariesConfiguration-$StressConfiguration.ps1"
$Runscript=".\run-stress-$StressConfiguration-$LibrariesConfiguration.ps1"
if (-not (Test-Path $Runscript)) {
Write-Host "Generating Runscript."
Add-Content -Path $Runscript -Value "& '$TestHostRoot/dotnet' exec --roll-forward Major ./bin/$StressConfiguration/net$Version/HttpStress.dll `$args"
}

Write-Host "To run tests type:"
Write-Host "$Runscript [stress test args]"
Write-Host "$Runscript [stress test args]"
1 change: 1 addition & 0 deletions src/libraries/System.Net.Http/tests/StressTests/HttpStress/windows.Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ARG VERSION=8.0
ARG CONFIGURATION=Release

RUN dotnet build -c $env:CONFIGURATION `
-p:MsQuicInteropIncludes="C:/live-runtime-artifacts/msquic-interop/*.cs" `
-p:TargetingPacksTargetsLocation=C:/live-runtime-artifacts/targetingpacks.targets `
-p:MicrosoftNetCoreAppRefPackDir=C:/live-runtime-artifacts/microsoft.netcore.app.ref/ `
-p:MicrosoftNetCoreAppRuntimePackDir=C:/live-runtime-artifacts/microsoft.netcore.app.runtime.win-x64/$env:CONFIGURATION/
Expand Down
4 changes: 2 additions & 2 deletions src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ static MsQuicApi()
}
string? gitHash = Marshal.PtrToStringUTF8((IntPtr)libGitHash);

MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} version={version} commit={gitHash}";
MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {version} ({gitHash})";

if (version < s_minMsQuicVersion)
{
Expand All @@ -143,7 +143,7 @@ static MsQuicApi()

if (NetEventSource.Log.IsEnabled())
{
NetEventSource.Info(null, $"Loaded MsQuic library version '{version}', commit '{gitHash}'.");
NetEventSource.Info(null, $"Loaded MsQuic library '{MsQuicLibraryVersion}'.");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Brilliant 😆

}

// Assume SChannel is being used on windows and query for the actual provider from the library if querying is supported
Expand Down
2 changes: 1 addition & 1 deletion src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ public async Task ConnectWithServerCertificateCallback()
// TODO: the exception may change if we implement https://github.com/dotnet/runtime/issues/73152 to make server close
// connections with CONNECTION_REFUSED in such cases
var authEx = await Assert.ThrowsAsync<AuthenticationException>(() => clientTask);
Assert.Contains("UserCanceled", authEx.Message);
Assert.Contains(TlsAlertMessage.UserCanceled.ToString(), authEx.Message);
Assert.Equal(clientOptions.ClientAuthenticationOptions.TargetHost, receivedHostName);
await Assert.ThrowsAsync<ArgumentException>(async () => await listener.AcceptConnectionAsync());

Expand Down
12 changes: 9 additions & 3 deletions src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
using Xunit.Abstractions;
using System.Diagnostics.Tracing;
using System.Net.Sockets;
using System.Reflection;
using Microsoft.Quic;
using static Microsoft.Quic.MsQuic;

namespace System.Net.Quic.Tests
{
Expand Down Expand Up @@ -44,14 +44,20 @@ public abstract class QuicTestBase : IDisposable

static unsafe QuicTestBase()
{
Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{MsQuicApi.MsQuicLibraryVersion}'.");
// If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if this should go somewhere to Common/test/.... so we can avoid duplication.
It may be worth of adding explanation as this is unusual craft.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds good, I'll look into it.

Copy link
Member

@antonfirsov antonfirsov Apr 25, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should keep in mind that whatever utility helper we add, its' file has to be also copied to the container, so avoiding duplication has a higher cost in complexity here than normally. At the moment I'm undecided if it's worth it, would prefer to do the docker copying work mentioned in #84793 (comment) first, planning to jump to it tomorrow.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I'm holding back. Let me know when/if I should look into it.

Type msQuicApiType = typeof(QuicConnection).Assembly.GetType("System.Net.Quic.MsQuicApi");

string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'.");

if (IsSupported)
{
object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty<object?>());
QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty<object?>())));
QUIC_SETTINGS settings = default(QUIC_SETTINGS);
settings.IsSet.MaxWorkerQueueDelayUs = 1;
settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default
if (StatusFailed(MsQuicApi.Api.ApiTable->SetParam(null, QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings)))
{
Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs.");
}
Expand Down
13 changes: 2 additions & 11 deletions src/libraries/System.Net.Quic/tests/FunctionalTests/System.Net.Quic.Functional.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
<Compile Include="$(CommonPath)System\Net\ArrayBuffer.cs" Link="ProductionCode\Common\System\Net\ArrayBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\MultiArrayBuffer.cs" Link="ProductionCode\Common\System\Net\MultiArrayBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\StreamBuffer.cs" Link="ProductionCode\Common\System\Net\StreamBuffer.cs" />
<Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
<Compile Include="$(CommonTestPath)System\IO\ConnectedStreams.cs" Link="Common\System\IO\ConnectedStreams.cs" />
<Compile Include="$(CommonTestPath)System\Net\Capability.Security.cs" Link="Common\System\Net\Capability.Security.cs" />
<Compile Include="$(CommonTestPath)System\Net\Configuration.cs" Link="Common\System\Net\Configuration.cs" />
Expand All @@ -31,18 +32,8 @@
</ItemGroup>
<!-- Shared production code -->
<ItemGroup>
<Compile Include="$(CommonPath)System\Net\Logging\NetEventSource.Common.cs" Link="Common\System\Net\Logging\NetEventSource.Common.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicApi.NativeMethods.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\MsQuicSafeHandle.cs" Link="ProductionCode\System\Net\Quic\Internal\MsQuicSafeHandle.cs" />
<Compile Include="..\..\src\System\Net\Quic\Internal\ThrowHelper.cs" Link="ProductionCode\System\Net\Quic\Internal\ThrowHelper.cs" />
<Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" Link="ProductionCode\System\Net\Quic\Interop\*.cs" />
<Compile Include="..\..\src\System\Net\Quic\NetEventSource.Quic.cs" Link="ProductionCode\System\Net\Quic\NetEventSource.Quic.cs" />
<Compile Include="..\..\src\System\Net\Quic\QuicDefaults.cs" Link="ProductionCode\System\Net\Quic\QuicDefaults.cs" />
<Compile Include="$(CommonPath)Interop\Windows\Interop.Libraries.cs" Link="Common\Interop\Windows\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'windows'" />
<Compile Include="$(CommonPath)Interop\Linux\Interop.Libraries.cs" Link="Common\Interop\Linux\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'linux'" />
<Compile Include="$(CommonPath)Interop\OSX\Interop.Libraries.cs" Link="Common\Interop\OSX\Interop.Libraries.cs" Condition="'$(TargetPlatformIdentifier)' == 'osx'" />
<Compile Include="$(CommonPath)System\Net\Security\TlsAlertMessage.cs" Link="Common\System\Net\Security\TlsAlertMessage.cs" />
<Compile Include="..\..\src\System\Net\Quic\Interop\*.cs" LinkBase="ProductionCode\System\Net\Quic\Interop" />
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed this to use LinkBase, Visual Studio is going crazy when Link is being used with wildcards:

image

</ItemGroup>
<ItemGroup>
<ProjectReference Include="$(CommonTestPath)StreamConformanceTests\StreamConformanceTests.csproj" />
Expand Down