Disable most networking tests for default CI runs

Lately, the CI system and test server infrastructure has been overloaded with many PRs. This is causing many networking tests to randomly fail due to resource exhaustion caused by so many simultaneous PR runs on the same CI machines. This occurrs not only when using an external test server endpoint (corefx-net.cloudapp.net) but also loopback servers.

To improve engineering efficiency for the team, we are moving most of the tests in the networking libraries from the default CI runs (innerloop). For developers working on networking libraries, the networking tests can be run by invoking the "outerloop" tests.

We will not be allowing any merges of PRs in the networking libraries unless the outerloop tests are also run by the developer.

This affects tests in the following libraries right now:

* System.Net.Http
* System.Net.Http.WinHttpHandler
* System.Net.Sockets
* System.Net.Security
* System.Net.WebSockets.Client

I will be working on fine tuning these exclusions (issue #11345) to try to bring back tests into the innerloop runs when we have begun to address the infrastructure issues.

src/System.Net.Http.WinHttpHandler/tests/FunctionalTests/ServerCertificateTest.cs

@@ -28,6 +28,7 @@ public ServerCertificateTest(ITestOutputHelper output)
             _validationCallbackHistory = new ValidationCallbackHistory();
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task NoCallback_ValidCertificate_CallbackNotCalled()
         {
@@ -40,6 +41,7 @@ public async Task NoCallback_ValidCertificate_CallbackNotCalled()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
         {
@@ -53,6 +55,7 @@ public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback()
         {
@@ -68,6 +71,7 @@ public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task UseCallback_RedirectandValidCertificate_ExpectedValuesDuringCallback()
         {
@@ -85,6 +89,7 @@ public async Task UseCallback_RedirectandValidCertificate_ExpectedValuesDuringCa
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task UseCallback_CallbackReturnsFailure_ThrowsInnerSecurityFailureException()
         {
@@ -103,6 +108,7 @@ public async Task UseCallback_CallbackReturnsFailure_ThrowsInnerSecurityFailureE
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task UseCallback_CallbackThrowsSpecificException_ThrowsInnerSpecificException()
         {

src/System.Net.Http.WinHttpHandler/tests/FunctionalTests/WinHttpHandlerTest.cs

@@ -30,6 +30,7 @@ public WinHttpHandlerTest(ITestOutputHelper output)
             _output = output;
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public void SendAsync_SimpleGet_Success()
         {
@@ -44,6 +45,7 @@ public void SendAsync_SimpleGet_Success()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Theory]
         [InlineData(CookieUsePolicy.UseInternalCookieStoreOnly, "cookieName1", "cookieValue1")]
         [InlineData(CookieUsePolicy.UseSpecifiedCookieContainer, "cookieName2", "cookieValue2")]
@@ -75,6 +77,7 @@ public async Task GetAsync_RedirectResponseHasCookie_CookieSentToFinalUri(
             }            
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         [OuterLoop]
         public async Task SendAsync_SlowServerAndCancel_ThrowsTaskCanceledException()
@@ -93,6 +96,7 @@ public async Task SendAsync_SlowServerAndCancel_ThrowsTaskCanceledException()
             }
         }        
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         [OuterLoop]
         public void SendAsync_SlowServerRespondsAfterDefaultReceiveTimeout_ThrowsHttpRequestException()

src/System.Net.Http/tests/FunctionalTests/DefaultCredentialsTest.cs

@@ -40,6 +40,7 @@ public DefaultCredentialsTest(ITestOutputHelper output)
             _output.WriteLine(s_authenticatedServer.ToString());
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -56,6 +57,7 @@ public async Task UseDefaultCredentials_DefaultValue_Unauthorized(bool useProxy)
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -71,6 +73,7 @@ public async Task UseDefaultCredentials_SetFalse_Unauthorized(bool useProxy)
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -91,6 +94,7 @@ public async Task UseDefaultCredentials_SetTrue_ConnectAsCurrentIdentity(bool us
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -111,6 +115,7 @@ public async Task UseDefaultCredentials_SetTrueAndServerOffersMultipleSchemes_Ok
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -132,6 +137,7 @@ public async Task Credentials_SetToSpecificCredential_ConnectAsSpecificIdentity(
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalTheory(nameof(DomainJoinedTestsEnabled))]
         [InlineData(false)]
@@ -157,6 +163,7 @@ public async Task Credentials_SetToWrappedDefaultCredential_ConnectAsCurrentIden
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalFact(nameof(DomainProxyTestsEnabled))]
         public async Task Proxy_UseAuthenticatedProxyWithNoCredentials_ProxyAuthenticationRequired()
@@ -171,6 +178,7 @@ public async Task Proxy_UseAuthenticatedProxyWithNoCredentials_ProxyAuthenticati
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(10041)]
         [ConditionalFact(nameof(DomainProxyTestsEnabled))]
         public async Task Proxy_UseAuthenticatedProxyWithDefaultCredentials_OK()
@@ -185,6 +193,7 @@ public async Task Proxy_UseAuthenticatedProxyWithDefaultCredentials_OK()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(DomainProxyTestsEnabled))]
         public async Task Proxy_UseAuthenticatedProxyWithWrappedDefaultCredentials_OK()
         {

src/System.Net.Http/tests/FunctionalTests/DiagnosticsTests.cs

@@ -25,6 +25,7 @@ public class DiagnosticsTest : RemoteExecutorTestBase
         /// This test must be in the same test collection as any others testing HttpClient/WinHttpHandler
         /// DiagnosticSources, since the global logging mechanism makes them conflict inherently.
         /// </remarks>
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public void SendAsync_ExpectedDiagnosticSourceLogging()
         {
@@ -79,6 +80,7 @@ public void SendAsync_ExpectedDiagnosticSourceLogging()
         /// This test must be in the same test collection as any others testing HttpClient/WinHttpHandler
         /// DiagnosticSources, since the global logging mechanism makes them conflict inherently.
         /// </remarks>
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public void SendAsync_ExpectedDiagnosticSourceNoLogging()
         {
@@ -113,6 +115,7 @@ public void SendAsync_ExpectedDiagnosticSourceNoLogging()
             }).Dispose();
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public void SendAsync_HttpTracingEnabled_Succeeds()
         {

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.ClientCertificates.cs

@@ -46,6 +46,7 @@ public void ClientCertificateOptions_ValueArg_Roundtrips(ClientCertificateOption
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
         public async Task Automatic_SSLBackendNotSupported_ThrowsPlatformNotSupportedException()
         {
@@ -55,6 +56,7 @@ public async Task Automatic_SSLBackendNotSupported_ThrowsPlatformNotSupportedExc
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
         public async Task Manual_SSLBackendNotSupported_ThrowsPlatformNotSupportedException()
         {
@@ -66,6 +68,7 @@ public async Task Manual_SSLBackendNotSupported_ThrowsPlatformNotSupportedExcept
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(9543, PlatformID.Windows)] // reuseClient==false fails in debug/release, reuseClient==true fails sporadically in release
         [ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
         [InlineData(6, false)]

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.DefaultProxyCredentials.cs

@@ -35,6 +35,7 @@ public void SetGet_Roundtrips()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public void ProxyExplicitlyProvided_DefaultCredentials_Ignored()
         {
@@ -63,6 +64,7 @@ public void ProxyExplicitlyProvided_DefaultCredentials_Ignored()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         [PlatformSpecific(PlatformID.AnyUnix)] // proxies set via the http_proxy environment variable are specific to Unix
         public void ProxySetViaEnvironmentVariable_DefaultProxyCredentialsUsed()

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.MaxResponseHeadersLength.cs

@@ -59,6 +59,7 @@ public async Task SetAfterUse_Throws()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Theory]
         [InlineData("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", 37, false)]
         [InlineData("HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n", 38, true)]

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.ServerCertificates.cs

@@ -15,6 +15,7 @@ namespace System.Net.Http.Functional.Tests
 {
     public class HttpClientHandler_ServerCertificates_Test
     {
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task NoCallback_ValidCertificate_CallbackNotCalled()
         {
@@ -34,6 +35,7 @@ public async Task NoCallback_ValidCertificate_CallbackNotCalled()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
         public async Task UseCallback_NotSecureConnection_CallbackNotCalled()
         {
@@ -67,6 +69,7 @@ public static IEnumerable<object[]> UseCallback_ValidCertificate_ExpectedValuesD
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
         [MemberData(nameof(UseCallback_ValidCertificate_ExpectedValuesDuringCallback_Urls))]
         public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback(Uri url, bool checkRevocation)
@@ -95,6 +98,7 @@ public async Task UseCallback_ValidCertificate_ExpectedValuesDuringCallback(Uri
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
         public async Task UseCallback_CallbackReturnsFailure_ThrowsException()
         {
@@ -106,6 +110,7 @@ public async Task UseCallback_CallbackReturnsFailure_ThrowsException()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
         public async Task UseCallback_CallbackThrowsException_ExceptionPropagates()
         {
@@ -125,6 +130,7 @@ public async Task UseCallback_CallbackThrowsException_ExceptionPropagates()
             new object[] { Configuration.Http.WrongHostNameCertRemoteServer },
         };
 
+        [OuterLoop] // TODO: Issue #11345
         [Theory]
         [MemberData(nameof(CertificateValidationServers))]
         public async Task NoCallback_BadCertificate_ThrowsException(string url)
@@ -135,6 +141,7 @@ public async Task NoCallback_BadCertificate_ThrowsException(string url)
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Fact]
         public async Task NoCallback_RevokedCertificate_NoRevocationChecking_Succeeds()
         {
@@ -145,6 +152,7 @@ public async Task NoCallback_RevokedCertificate_NoRevocationChecking_Succeeds()
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsCustomCertificateHandling))]
         public async Task NoCallback_RevokedCertificate_RevocationChecking_Fails()
         {
@@ -162,6 +170,7 @@ public async Task NoCallback_RevokedCertificate_RevocationChecking_Fails()
             new object[] { Configuration.Http.WrongHostNameCertRemoteServer , SslPolicyErrors.RemoteCertificateNameMismatch},
         };
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(7812, PlatformID.Windows)]
         [ConditionalTheory(nameof(BackendSupportsCustomCertificateHandling))]
         [MemberData(nameof(CertificateValidationServersAndExpectedPolicies))]
@@ -191,6 +200,7 @@ public async Task UseCallback_BadCertificate_ExpectedPolicyErrors(string url, Ss
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
         public async Task SSLBackendNotSupported_Callback_ThrowsPlatformNotSupportedException()
         {
@@ -200,6 +210,7 @@ public async Task SSLBackendNotSupported_Callback_ThrowsPlatformNotSupportedExce
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendDoesNotSupportCustomCertificateHandling))]
         public async Task SSLBackendNotSupported_Revocation_ThrowsPlatformNotSupportedException()
         {
@@ -209,6 +220,7 @@ public async Task SSLBackendNotSupported_Revocation_ThrowsPlatformNotSupportedEx
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [PlatformSpecific(PlatformID.Windows)] // CopyToAsync(Stream, TransportContext) isn't used on unix
         [Fact]
         public async Task PostAsync_Post_ChannelBinding_ConfiguredCorrectly()

src/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.SslProtocols.cs

@@ -43,6 +43,7 @@ public void SetGetProtocols_Roundtrips(SslProtocols protocols)
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsSslConfiguration))]
         public async Task SetProtocols_AfterRequest_ThrowsException()
         {
@@ -59,6 +60,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [Theory]
         [InlineData(~SslProtocols.None)]
 #pragma warning disable 0618 // obsolete warning
@@ -75,6 +77,7 @@ public void DisabledProtocols_SetSslProtocols_ThrowsException(SslProtocols disab
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalTheory(nameof(BackendSupportsSslConfiguration))]
         [InlineData(SslProtocols.Tls, false)]
         [InlineData(SslProtocols.Tls, true)]
@@ -144,6 +147,7 @@ public async Task GetAsync_UnsupportedSSLVersion_Throws(string name, string url)
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalFact(nameof(BackendSupportsSslConfiguration), nameof(SslDefaultsToTls12))]
         public async Task GetAsync_NoSpecifiedProtocol_DefaultsToTls12()
         {
@@ -165,6 +169,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ConditionalTheory(nameof(BackendSupportsSslConfiguration))]
         [InlineData(SslProtocols.Tls11, SslProtocols.Tls, typeof(IOException))]
         [InlineData(SslProtocols.Tls12, SslProtocols.Tls11, typeof(IOException))]
@@ -185,6 +190,7 @@ await TestHelper.WhenAllCompletedOrAnyFailed(
             }
         }
 
+        [OuterLoop] // TODO: Issue #11345
         [ActiveIssue(8538, PlatformID.Windows)]
         [Fact]
         public async Task GetAsync_DisallowTls10_AllowTls11_AllowTls12()