Backport 17.6.2 fixes

eng/Versions.props

@@ -83,8 +83,10 @@
     <VSTestConsoleMostDownloadedVersion>[16.6.1]</VSTestConsoleMostDownloadedVersion>
     <VSTestConsolePreviousStableVersion>[16.11.0]</VSTestConsolePreviousStableVersion>
     <VSTestConsoleLegacyStableVersion>[15.9.2]</VSTestConsoleLegacyStableVersion>
-    <!-- This version also needs to be updated in src\package\nuspec\TestPlatform.ObjectModel.nuspec -->
-    <NuGetFrameworksVersion>5.11.0</NuGetFrameworksVersion>
+    <!-- This version also needs to be updated in src\Microsoft.TestPlatform.ObjectModel\Microsoft.TestPlatform.ObjectModel.nuspec -->
+    <!-- This version needs to be the same or lower than <NuGetFrameworksPackageVersion> https://github.com/dotnet/sdk/blob/release/main/eng/Versions.props
+    (or the respective branch. It also needs to be on nuget.org. -->
+    <NuGetFrameworksVersion>6.5.0</NuGetFrameworksVersion>
     <ILAsmPackageVersion>5.0.0</ILAsmPackageVersion>
     <JsonNetVersion>13.0.1</JsonNetVersion>
   </PropertyGroup>

src/Microsoft.TestPlatform.ObjectModel/Microsoft.TestPlatform.ObjectModel.nuspec

@@ -6,17 +6,17 @@
     <dependencies>
       <group targetFramework="net462">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
 
       <group targetFramework="netcoreapp3.1">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
 
       <group targetFramework="netstandard2.0">
         <dependency id="System.Reflection.Metadata" version="[1.6.0, )" />
-        <dependency id="NuGet.Frameworks" version="[5.11.0, )" />
+        <dependency id="NuGet.Frameworks" version="[6.5.0, )" />
       </group>
     </dependencies>
 

src/vstest.console/CommandLine/CommandLineOptions.cs

@@ -8,6 +8,7 @@
 using System.Linq;
 
 using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors;
+using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
 using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
@@ -287,7 +288,8 @@ public void AddSource(string source)
                 string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex);
         }
         // Add the matching files to source list
-        _sources = _sources.Union(matchingFiles).ToList();
+        var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles);
+        _sources = _sources.Union(filteredFiles).ToList();
     }
 
     /// <summary>

src/vstest.console/TestPlatformHelpers/TestRequestManager.cs

@@ -166,7 +166,7 @@ public void DiscoverTests(
         EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started.");
 
         // TODO: Normalize rest of the data on the request as well
-        discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List<string>();
+        discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList());
         discoveryPayload.RunSettings ??= "<RunSettings></RunSettings>";
 
         var runsettings = discoveryPayload.RunSettings;
@@ -274,6 +274,11 @@ public void RunTests(
     {
         EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
         testRunRequestPayload.RunSettings ??= "<RunSettings></RunSettings>";
+        if (testRunRequestPayload.Sources != null)
+        {
+            testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources);
+        }
+
         var runsettings = testRunRequestPayload.RunSettings;
 
         if (testRunRequestPayload.TestPlatformOptions != null)
@@ -1464,3 +1469,73 @@ private static List<string> GetSources(TestRunRequestPayload testRunRequestPaylo
         return sources;
     }
 }
+
+internal static class KnownPlatformSourceFilter
+{
+    // Running tests on AzureDevops, many projects use the default filter
+    // which includes all *test*.dll, this includes many of the TestPlatform dlls,
+    // which we cannot run, and don't want to attempt to run.
+    // The default filter also filters out !*TestAdapter*.dll but it is easy to forget
+    // so we skip the most used adapters here as well.
+    private static readonly HashSet<string> KnownPlatformSources = new(new string[]
+    {
+        "Microsoft.TestPlatform.AdapterUtilities.dll",
+        "Microsoft.TestPlatform.AdapterUtilities.resources.dll",
+        "Microsoft.TestPlatform.CommunicationUtilities.dll",
+        "Microsoft.TestPlatform.CommunicationUtilities.resources.dll",
+        "Microsoft.TestPlatform.CoreUtilities.dll",
+        "Microsoft.TestPlatform.CoreUtilities.resources.dll",
+        "Microsoft.TestPlatform.CrossPlatEngine.dll",
+        "Microsoft.TestPlatform.CrossPlatEngine.resources.dll",
+        "Microsoft.TestPlatform.PlatformAbstractions.dll",
+        "Microsoft.TestPlatform.Utilities.dll",
+        "Microsoft.TestPlatform.Utilities.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.Common.dll",
+        "Microsoft.VisualStudio.TestPlatform.Common.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.ObjectModel.dll",
+        "Microsoft.VisualStudio.TestPlatform.ObjectModel.resources.dll",
+        "testhost.dll",
+
+        // NUnit
+        "NUnit3.TestAdapter.dll",
+
+        // XUnit
+        "xunit.runner.visualstudio.testadapter.dll",
+        "xunit.runner.visualstudio.dotnetcore.testadapter.dll",
+
+        // MSTest
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.dll",
+        "Microsoft.VisualStudio.TestPlatform.TestFramework.resources.dll",
+        // For MSTest up to v3
+        "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.resources.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll",
+        "Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.resources.dll",
+    }, StringComparer.OrdinalIgnoreCase);
+
+
+    internal static List<string> FilterKnownPlatformSources(List<string>? sources)
+    {
+        if (sources == null)
+        {
+            return new List<string>();
+        }
+
+        var filteredSources = new List<string>();
+        foreach (string source in sources)
+        {
+            if (KnownPlatformSources.Contains(Path.GetFileName(source)))
+            {
+                EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'");
+            }
+            else
+            {
+                filteredSources.Add(source);
+            }
+        }
+
+        return filteredSources;
+    }
+}

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs

@@ -10,6 +10,8 @@
 
 using TestPlatform.TestUtilities;
 using System.Linq;
+using Microsoft.VisualStudio.TestPlatform.Common;
+using FluentAssertions;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;
 
@@ -420,4 +422,100 @@ public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(Runne
 
         ExitCodeEquals(1);
     }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource]
+    public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+        var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList();
+        fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed.");
+        fails.Last().Should().Be("Test Run Failed.");
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    [NetCoreTargetFrameworkDataSource()]
+    public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErrHasTestRunFailedMessageButNoOtherError();
+    }
+
+    [TestMethod]
+    [NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
+    public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo)
+    {
+        // This is the default filter of AzDo VSTest task:
+        //     testAssemblyVer2: |
+        //       **\*test *.dll
+        //       ! * *\*TestAdapter.dll
+        //       ! * *\obj\**
+        // Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
+        // or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
+        // or deps.json, and fails the run.
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
+        var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll"));
+
+        string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote()));
+        InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
+
+        StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out");
+    }
 }