Utility for uploading artifact on test failure

src/Compilers/Core/MSBuildTaskTests/DotNetSdkTests.cs

@@ -7,11 +7,17 @@
 using System.IO;
 using Roslyn.Test.Utilities;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Microsoft.CodeAnalysis.BuildTasks.UnitTests
 {
     public class DotNetSdkTests : DotNetSdkTestBase
     {
+        public DotNetSdkTests(ITestOutputHelper testOutputHelper)
+            : base(testOutputHelper)
+        {
+        }
+
         [ConditionalFact(typeof(DotNetSdkAvailable), AlwaysSkip = "https://github.com/dotnet/roslyn/issues/46304")]
         [WorkItem(22835, "https://github.com/dotnet/roslyn/issues/22835")]
         public void TestSourceLink()

src/Compilers/Core/MSBuildTaskTests/TestUtilities/DotNetSdkTestBase.cs

@@ -11,6 +11,7 @@
 using Microsoft.CodeAnalysis.Test.Utilities;
 using Roslyn.Test.Utilities;
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Microsoft.CodeAnalysis.BuildTasks.UnitTests
 {
@@ -46,6 +47,7 @@ public void F()
 }
 ";
 
+        protected readonly ITestOutputHelper TestOutputHelper;
         protected readonly TempDirectory ProjectDir;
         protected readonly TempDirectory ObjDir;
         protected readonly TempDirectory OutDir;
@@ -89,26 +91,31 @@ static bool isMatchingDotNetInstance(string? dotnetDir)
         private static void EmitTestHelperProps(
             string objDirectory,
             string projectFileName,
-            string? content)
+            string? content,
+            ArtifactUploadUtil? uploadUtil)
         {
             // Common.props automatically import {project-name}.*.props files from MSBuildProjectExtensionsPath directory, 
             // which is by default set to the IntermediateOutputPath:
-            File.WriteAllText(Path.Combine(objDirectory, projectFileName + ".TestHelpers.g.props"),
+            var filePath = Path.Combine(objDirectory, projectFileName + ".TestHelpers.g.props");
+            File.WriteAllText(filePath,
 $@"<Project>
 {content}
 </Project>");
+            uploadUtil?.AddArtifact(filePath);
         }
 
         private static void EmitTestHelperTargets(
             string objDirectory,
             string outputFile,
             string projectFileName,
             IEnumerable<string> expressions,
-            string? additionalContent)
+            string? additionalContent,
+            ArtifactUploadUtil? uploadUtil)
         {
             // Common.targets automatically import {project-name}.*.targets files from MSBuildProjectExtensionsPath directory, 
             // which is by defautl set to the IntermediateOutputPath:
-            File.WriteAllText(Path.Combine(objDirectory, projectFileName + ".TestHelpers.g.targets"),
+            var filePath = Path.Combine(objDirectory, projectFileName + ".TestHelpers.g.targets");
+            File.WriteAllText(filePath,
 $@"<Project>      
   <Target Name=""Test_EvaluateExpressions"">
     <PropertyGroup>
@@ -133,9 +140,11 @@ private static void EmitTestHelperTargets(
 
 {additionalContent}
 </Project>");
+
+            uploadUtil?.AddArtifact(filePath);
         }
 
-        public DotNetSdkTestBase()
+        public DotNetSdkTestBase(ITestOutputHelper testOutputHelper)
         {
             Assert.True(s_dotnetInstallDir is object, $"SDK not found. Use {nameof(ConditionalFactAttribute)}(typeof({nameof(DotNetSdkAvailable)})) to skip the test if the SDK is not found.");
             Debug.Assert(s_dotnetInstallDir is object);
@@ -144,6 +153,7 @@ public DotNetSdkTestBase()
             var testBinDirectory = Path.GetDirectoryName(typeof(DotNetSdkTests).Assembly.Location) ?? string.Empty;
             var sdksDir = Path.Combine(s_dotnetSdkPath ?? string.Empty, "Sdks");
 
+            TestOutputHelper = testOutputHelper;
             ProjectName = "test";
             ProjectFileName = ProjectName + ".csproj";
             Configuration = "Debug";
@@ -186,14 +196,16 @@ public DotNetSdkTestBase()
 
         protected void VerifyValues(string? customProps, string? customTargets, string[] targets, string[] expressions, string[] expectedResults)
         {
+            using var uploadUtil = new ArtifactUploadUtil(TestOutputHelper);
             var evaluationResultsFile = Path.Combine(OutDir.Path, "EvaluationResult.txt");
 
-            EmitTestHelperProps(ObjDir.Path, ProjectFileName, customProps);
-            EmitTestHelperTargets(ObjDir.Path, evaluationResultsFile, ProjectFileName, expressions, customTargets);
+            EmitTestHelperProps(ObjDir.Path, ProjectFileName, customProps, uploadUtil);
+            EmitTestHelperTargets(ObjDir.Path, evaluationResultsFile, ProjectFileName, expressions, customTargets, uploadUtil);
 
             var targetsArg = string.Join(";", targets.Concat(new[] { "Test_EvaluateExpressions" }));
             var testBinDirectory = Path.GetDirectoryName(typeof(DotNetSdkTests).Assembly.Location);
             var binLog = Path.Combine(ProjectDir.Path, $"build{_logIndex++}.binlog");
+            uploadUtil.AddArtifact(binLog);
 
             // RoslynTargetsPath is a path to the built-in Roslyn compilers in the .NET SDK.
             // For testing we are using compilers from custom location (this emulates usage of Microsoft.Net.Compilers package.
@@ -204,6 +216,7 @@ protected void VerifyValues(string? customProps, string? customTargets, string[]
 
             var evaluationResult = File.ReadAllLines(evaluationResultsFile).Select(l => (l != EmptyValueMarker) ? l : "");
             AssertEx.Equal(expectedResults, evaluationResult);
+            uploadUtil.SetSucceeded();
         }
     }
 }

src/Compilers/Test/Core/Assert/ArtifactUploadUtil.cs

@@ -0,0 +1,76 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit.Abstractions;
+
+namespace Roslyn.Test.Utilities
+{
+    /// <summary>
+    /// There are test failures that are virtually impossible to debug without artifacts generated during
+    /// the test execution. This utility is helpful at getting those artifacts attached to AzDO / Helix 
+    /// when executed in our CI process. 
+    /// 
+    /// The utilility works by collecting a set of file paths to artifacts. If the test succeeds it should 
+    /// call the <see cref="SetSucceeded"/> method. Otherwise if the test fails an exception is generated,
+    /// the call is skipped and in <see cref="Dispose"/> we prepare the artifacts for upload.
+    /// </summary>
+    public sealed class ArtifactUploadUtil : IDisposable
+    {
+        private readonly ITestOutputHelper _testOutputHelper;
+        private readonly string _baseDirectoryName;
+        private readonly List<string> _artifactFilePath = new List<string>();
+        private bool _success = false;
+
+        public ArtifactUploadUtil(ITestOutputHelper testOutputHelper, string? baseDirectoryName = null)
+        {
+            _testOutputHelper = testOutputHelper;
+            _baseDirectoryName = baseDirectoryName ?? Guid.NewGuid().ToString();
+        }
+
+        public void AddArtifact(string filePath)
+        {
+            _artifactFilePath.Add(filePath);
+        }
+
+        public void SetSucceeded()
+        {
+            _success = true;
+        }
+
+        public void Dispose()
+        {
+            if (!_success)
+            {
+                var uploadDir = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
+                if (string.IsNullOrEmpty(uploadDir))
+                {
+                    _testOutputHelper.WriteLine("Skipping artifact upload as not running in helix");
+                    _testOutputHelper.WriteLine("Artifacts");
+                    foreach (var filePath in _artifactFilePath)
+                    {
+                        _testOutputHelper.WriteLine(filePath);
+                    }
+                }
+                else
+                {
+                    uploadDir = Path.Combine(uploadDir, _baseDirectoryName);
+                    Directory.CreateDirectory(uploadDir);
+                    _testOutputHelper.WriteLine($"Uploading artifacts by copying to {uploadDir}");
+                    foreach (var filePath in _artifactFilePath)
+                    {
+                        _testOutputHelper.WriteLine($"Copying {filePath}");
+                        var fileName = Path.GetFileName(filePath);
+                        File.Copy(filePath, Path.Combine(uploadDir, fileName));
+                    }
+                }
+            }
+        }
+    }
+}