Increase timeouts in tests to improve pipelines stability (#25)

david-garcia-garcia · Jul 19, 2024 · cab1a71 · cab1a71
1 parent 5ae529c
commit cab1a71
Show file tree

Hide file tree

Showing 12 changed files with 38 additions and 33 deletions.
diff --git a/bootstraptest.ps1 b/bootstraptest.ps1
@@ -7,7 +7,7 @@ function OutputLog {
 
     $logs = Invoke-Command -Script {
         $ErrorActionPreference = "silentlycontinue"
-        docker logs $containerName --tail 225 2>&1
+        docker logs $containerName --tail 250 2>&1
     } -ErrorAction SilentlyContinue
     Write-Host "---------------- LOGSTART"
     Write-Host ($logs -join "`r`n")
@@ -18,17 +18,23 @@ function WaitForLog {
     param (
         [string]$containerName,
         [string]$logContains,
-        [int]$timeoutSeconds = 25
+        [switch]$extendedTimeout
     )
 
+    $timeoutSeconds = 20;
+
+    if ($extendedTimeout) {
+        $timeoutSeconds = 60;
+    }
+
     $timeout = New-TimeSpan -Seconds $timeoutSeconds
     $sw = [System.Diagnostics.Stopwatch]::StartNew()
 
     while ($sw.Elapsed -le $timeout) {
         Start-Sleep -Seconds 1
         $logs = Invoke-Command -Script {
             $ErrorActionPreference = "silentlycontinue"
-            docker logs $containerName --tail 150 2>&1
+            docker logs $containerName --tail 250 2>&1
         } -ErrorAction SilentlyContinue
         if ($logs -match $logContains) {
             return;

diff --git a/servercore2022/tests/Compose.Tests.ps1 b/servercore2022/tests/Compose.Tests.ps1
@@ -6,7 +6,7 @@ Describe 'compose.yaml' {
 
     It 'Container starts' {
         docker compose -f servercore2022/compose.yaml up -d;
-        WaitForLog $Env:imageName "Initialization Completed"
+        WaitForLog $Env:imageName "Initialization Completed" -extendedTimeout
     }
 
     It 'LogRotate runs at 5AM Daily' {

diff --git a/servercore2022/tests/ComposeAsync.Tests.ps1 b/servercore2022/tests/ComposeAsync.Tests.ps1
@@ -2,7 +2,7 @@ Describe 'compose-async.yaml' {
     BeforeAll {
         . ./../bootstraptest.ps1
         docker compose -f servercore2022/compose-async.yaml up -d;
-        WaitForLog "servercore2022-servercore-1" "Initialization Completed"
+        WaitForLog "servercore2022-servercore-1" "Initialization Completed"  -extendedTimeout
     }
 
     It 'Booted asynchronously' {

diff --git a/servercore2022/tests/ComposeBasic.Tests.ps1 b/servercore2022/tests/ComposeBasic.Tests.ps1
@@ -6,7 +6,7 @@ Describe 'compose-basic.yaml' {
 
     It 'Container initializes' {
         docker compose -f servercore2022/compose-basic.yaml up -d;
-        WaitForLog $Env:ImageName "Initialization Completed"
+        WaitForLog $Env:ImageName "Initialization Completed"  -extendedTimeout
     }
 
     It 'Booted synchronously' {
@@ -31,7 +31,7 @@ Describe 'compose-basic.yaml' {
 
     It 'Shutdown not called twice' {
         docker exec $Env:ImageName powershell "powershell -File c:\entrypoint\shutdown.ps1"
-        WaitForLog $Env:ImageName "SHUTDOWN END" -timeoutSeconds 20
+        WaitForLog $Env:ImageName "SHUTDOWN END" -extendedTimeout
         docker compose -f servercore2022/compose-basic.yaml stop;
         WaitForLog $Env:ImageName "Integrated shutdown skipped"
     }

diff --git a/servercore2022iis/tests/Compose.Tests.ps1 b/servercore2022iis/tests/Compose.Tests.ps1
@@ -2,7 +2,7 @@ Describe 'compose.yaml' {
     BeforeAll {
         . ./../bootstraptest.ps1
         docker compose -f servercore2022iis/compose.yaml up -d;
-        WaitForLog "servercore2022iis-web-1" "Initialization Completed"
+        WaitForLog "servercore2022iis-web-1" "Initialization Completed"  -extendedTimeout
     }
     It 'Responds on port 80 HTTP' {
         (Invoke-WebRequest 172.18.8.8 -UseBasicParsing).RawContent | Should -Match "iisstart\.png";

diff --git a/servercore2022iis/tests/ComposeCerts.Tests.ps1 b/servercore2022iis/tests/ComposeCerts.Tests.ps1
@@ -2,7 +2,7 @@ Describe 'compose.yaml' {
     BeforeAll {
         . ./../bootstraptest.ps1
         docker compose -f servercore2022iis/compose-certs.yaml up -d;
-        WaitForLog "servercore2022iis-web-1" "Initialization Completed"
+        WaitForLog "servercore2022iis-web-1" "Initialization Completed" -extendedTimeout
     }
 
     It 'HTTP Response OK with a hostname from SBS_IISBINDINGS' {

diff --git a/sqlserver2022base/tests/Compose.Tests.ps1 b/sqlserver2022base/tests/Compose.Tests.ps1
@@ -9,7 +9,7 @@ Describe 'compose.yaml' {
 
     It 'Server starts' {
         docker compose -f sqlserver2022base/compose.yaml up -d;
-        WaitForLog $Env:containerName "Initialization Completed" -TimeoutSeconds 30
+        WaitForLog $Env:containerName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {

diff --git a/sqlserver2022k8s/tests/Compose-backups-bacpack.Tests.ps1 b/sqlserver2022k8s/tests/Compose-backups-bacpack.Tests.ps1
@@ -10,7 +10,7 @@ Describe 'compose-backups.yaml' {
 
     It 'SQL Server starts' {
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 30
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {
@@ -78,8 +78,7 @@ CREATE TABLE dbo.TestTable (
         # Restore from bacpac using SbsRestoreFull
         docker exec $Env:instanceName powershell "Import-Module Sbs;Import-Module dbatools;SbsRestoreFull -SqlInstance localhost -DatabaseName renamedDatabase2 -Path '$($containerPath)'"
         Get-DbaDatabase -SqlInstance $Env:connectionString -Database renamedDatabase2 | Should -Not -BeNullOrEmpty;
-        WaitForLog $Env:instanceName "Restored database from" -TimeoutSeconds 10
-
+        WaitForLog $Env:instanceName "Restored database from" -extendedTimeout
         # Test that the database has the table we created before
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database renamedDatabase2 -Query "SELECT OBJECT_ID('dbo.TestTable')").Column1 | Should -Not -BeNullOrEmpty
     }

diff --git a/sqlserver2022k8s/tests/Compose-backups.Tests.ps1 b/sqlserver2022k8s/tests/Compose-backups.Tests.ps1
@@ -10,7 +10,7 @@ Describe 'compose-backups.yaml' {
 
     It 'SQL Server starts' {
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 30
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {
@@ -36,8 +36,8 @@ CREATE TABLE dbo.TestTable (
     It 'Tear down makes backups' {
         # Decommission the docker
         docker compose -f sqlserver2022k8s/compose-backups.yaml stop
-        WaitForLog $Env:instanceName "Performing shutdown backups" -TimeoutSeconds 30;
-        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -TimeoutSeconds 30;
+        WaitForLog $Env:instanceName "Performing shutdown backups" -extendedTimeout
+        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -extendedTimeout
         docker compose -f sqlserver2022k8s/compose-backups.yaml down
     }
 
@@ -50,7 +50,7 @@ CREATE TABLE dbo.TestTable (
     It 'Backups are recovered' {
         # Start the container again
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 30;
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
 
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT OBJECT_ID('dbo.TestTable')").Column1 | Should -Not -BeNullOrEmpty
 
@@ -66,13 +66,13 @@ CREATE TABLE dbo.TestTable (
     It 'Tear down makes backups' {
         # This shutdown adds 1 trn file
         docker compose -f sqlserver2022k8s/compose-backups.yaml stop
-        WaitForLog $Env:instanceName "Performing shutdown backups" -TimeoutSeconds 30;
-        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -TimeoutSeconds 30;
+        WaitForLog $Env:instanceName "Performing shutdown backups" -extendedTimeout
+        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -extendedTimeout
         docker compose -f sqlserver2022k8s/compose-backups.yaml down
 
         # This cycle adds an additional trn file
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog "sqlserver2022k8s-mssql-1" "Initialization Completed" -TimeoutSeconds 30;
+        WaitForLog "sqlserver2022k8s-mssql-1" "Initialization Completed" -extendedTimeout
 
         $insertQuery = @"
         INSERT INTO dbo.TestTable (TestData)
@@ -85,8 +85,8 @@ CREATE TABLE dbo.TestTable (
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 2").TestData | Should -Be "New Record 2"
 
         docker compose -f sqlserver2022k8s/compose-backups.yaml stop
-        WaitForLog $Env:instanceName "Performing shutdown backups" -TimeoutSeconds 30;
-        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -TimeoutSeconds 30;
+        WaitForLog $Env:instanceName "Performing shutdown backups" -extendedTimeout
+        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -extendedTimeout
         docker compose -f sqlserver2022k8s/compose-backups.yaml down
     }
 
@@ -98,17 +98,17 @@ CREATE TABLE dbo.TestTable (
 
     It 'Information is recovered after several cycles of backups and restores' {
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 40;
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 1").TestData | Should -Be "New Record"
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 2").TestData | Should -Be "New Record 2"
         docker compose -f sqlserver2022k8s/compose-backups.yaml down
     }
 
     It 'Can make a diff backup' {
         docker compose -f sqlserver2022k8s/compose-backups.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 40;
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
         docker exec $Env:instanceName powershell "SbsMssqlRunBackups DIFF";
-        WaitForLog $Env:instanceName "backups finished" -TimeoutSeconds 90;
+        WaitForLog $Env:instanceName "backups finished" -extendedTimeout
         $backupFiles = Get-ChildItem -Path "$env:BUILD_TEMP\datavolume\backup\mytestdatabase\DIFF" -Recurse -Filter "*.bak"
         $backupFiles.Count | Should -Be 1
     }

diff --git a/sqlserver2022k8s/tests/Compose-jobs.Tests.ps1 b/sqlserver2022k8s/tests/Compose-jobs.Tests.ps1
@@ -11,7 +11,7 @@ Describe 'compose-jobs.yaml' {
 
     It 'SQL Server starts' {
         docker compose -f sqlserver2022k8s/compose-jobs.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 30
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {

diff --git a/sqlserver2022k8s/tests/Compose-persistent.Tests.ps1 b/sqlserver2022k8s/tests/Compose-persistent.Tests.ps1
@@ -10,7 +10,7 @@ Describe 'compose-persistent.yaml' {
 
     It 'SQL Server starts' {
         docker compose -f sqlserver2022k8s/compose-persistent.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 40
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {
@@ -41,7 +41,7 @@ CREATE TABLE dbo.TestTable (
 
     It 'Can start' {
         docker compose -f sqlserver2022k8s/compose-persistent.yaml up -d;
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 25;
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
     }
 
     It 'State is preserved' {
@@ -58,12 +58,12 @@ CREATE TABLE dbo.TestTable (
     It 'Tear down works' {
         # This shutdown adds 1 trn file
         docker compose -f sqlserver2022k8s/compose-persistent.yaml stop
-        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -TimeoutSeconds 15;
+        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -extendedTimeout
         docker compose -f sqlserver2022k8s/compose-persistent.yaml down
 
         # This cycle adds an additional trn file
         docker compose -f sqlserver2022k8s/compose-persistent.yaml up -d
-        WaitForLog "sqlserver2022k8s-mssql-1" "Initialization Completed" -TimeoutSeconds 25;
+        WaitForLog "sqlserver2022k8s-mssql-1" "Initialization Completed" -extendedTimeout
 
         $insertQuery = @"
         INSERT INTO dbo.TestTable (TestData)
@@ -76,13 +76,13 @@ CREATE TABLE dbo.TestTable (
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 2").TestData | Should -Be "New Record 2"
 
         docker compose -f sqlserver2022k8s/compose-persistent.yaml stop
-        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -TimeoutSeconds 15;
+        WaitForLog $Env:instanceName "Entry point SHUTDOWN END" -extendedTimeout
         docker compose -f sqlserver2022k8s/compose-persistent.yaml down
     }
 
     It 'Information is recovered after several cycles of backups and restores' {
         docker compose -f sqlserver2022k8s/compose-persistent.yaml up -d
-        WaitForLog $Env:instanceName "Initialization Completed" -TimeoutSeconds 25;
+        WaitForLog $Env:instanceName "Initialization Completed" -extendedTimeout
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 1").TestData | Should -Be "New Record"
         (Invoke-DbaQuery -SqlInstance $Env:connectionString -Database mytestdatabase -Query "SELECT TestData FROM dbo.TestTable WHERE ID = 2").TestData | Should -Be "New Record 2"
         docker compose -f sqlserver2022k8s/compose-persistent.yaml down

diff --git a/sqlserver2022k8s/tests/Compose.Tests.ps1 b/sqlserver2022k8s/tests/Compose.Tests.ps1
@@ -6,7 +6,7 @@ Describe 'compose.yaml' {
         $Env:connectionString = "Server=172.18.8.8;User Id=sa;Password=sapwd;";
         $Env:containerName = "sqlserver2022k8s-mssql-1"
         docker compose -f sqlserver2022k8s/compose.yaml up -d;
-        WaitForLog $Env:containerName "Initialization Completed" -TimeoutSeconds 30
+        WaitForLog $Env:containerName "Initialization Completed" -extendedTimeout
     }
 
     It 'Can connect to the SQL Server' {