Make failure to call blob.exists() warn instead of throw for a known …

…error mode. In most cases this exception was not fatal; this only impacts users who upload the same output path multiple times in a helix job and in a vast majority of such cases the first upload does succeed. (#11364)

src/Microsoft.DotNet.Helix/JobSender/StorageHelpers/ContainerBase.cs

@@ -25,7 +25,7 @@ public async Task<Uri> UploadFileAsync(Stream stream, string blobName, Action<st
             }
             catch (RequestFailedException e) when (e.Status == 409)
             {
-                if (!pageBlob.Exists())
+                if (!await CheckExistenceWithRetry(pageBlob, log))
                 {
                     log?.Invoke($"error : Upload of {pageBlob.Uri} failed with {e.ErrorCode}, but the blob does not exist.");
                     throw;
@@ -47,6 +47,31 @@ public async Task<Uri> UploadTextAsync(string text, string blobName, Action<stri
             return await UploadFileAsync(new MemoryStream(bytes), blobName, log, cancellationToken);
         }
 
+        private async Task<bool> CheckExistenceWithRetry(BlobClient blobClient, Action<string> log)
+        {
+            int attemptsRemaining = 5;
+            while (attemptsRemaining > 0)
+            {
+                try
+                {
+                    bool result = await blobClient.ExistsAsync();
+                    return result;
+                }
+                // We hit these known issues with the Azure.Storage.Blobs library:
+                // https://github.com/Azure/azure-storage-net/issues/1040
+                // https://github.com/Azure/azure-storage-net/issues/1012
+                // Since these are currently not addressed and this is a fallback behavior
+                // (checking if something's been uploaded) we'll warn and return true if the service hits this repeatedly.
+                catch (RequestFailedException ex) when (ex.Status == 403)
+                {
+                    attemptsRemaining--;
+                    await Task.Delay(1000);
+                }
+            }
+            log?.Invoke($"warning : Failed to check existence of {blobClient.Uri} but the blob likely exists, continuing.");
+            return true;
+        }
+
         public abstract string Uri { get; }
         public abstract string ReadSas { get; }
         public abstract string WriteSas { get; }