Add test suite that uses toxiproxy

Fixes #1492 * Add dedicated test class for Toxiproxy * Start Toxiproxy via docker on GHA ubuntu runner * Start Toxiproxy in same Powershell session as `dotnet test` on GHA windows runner * Add toxiproxy-netcore as a submodule so that we can strong-name it. * Remove use of socket read timeout, and depend on heartbeats instead. * Remove duplicate tests * Use `ConcurrentBag` where necessary
rabbitmq · Feb 19, 2024 · eeaf91b · eeaf91b
1 parent 16bfe25
commit eeaf91b
Show file tree

Hide file tree

Showing 20 changed files with 296 additions and 101 deletions.
diff --git a/.ci/ubuntu/gha-setup.sh b/.ci/ubuntu/gha-setup.sh
@@ -9,6 +9,7 @@ readonly script_dir
 echo "[INFO] script_dir: '$script_dir'"
 
 readonly docker_name_prefix='rabbitmq-dotnet-client'
+readonly docker_network_name="$docker_name_prefix-network"
 
 if [[ ! -v GITHUB_ACTIONS ]]
 then
@@ -23,18 +24,46 @@ else
     echo "[INFO] set GITHUB_WORKSPACE to: '$GITHUB_WORKSPACE'"
 fi
 
+if [[ $1 == 'toxiproxy' ]]
+then
+    readonly run_toxiproxy='true'
+else
+    readonly run_toxiproxy='false'
+fi
+
 set -o nounset
 
 declare -r rabbitmq_docker_name="$docker_name_prefix-rabbitmq"
+declare -r toxiproxy_docker_name="$docker_name_prefix-toxiproxy"
+
+function start_toxiproxy
+{
+    if [[ $run_toxiproxy == 'true' ]]
+    then
+        sudo ss -4nlp
+        echo "[INFO] starting Toxiproxy server docker container"
+        docker rm --force "$toxiproxy_docker_name" 2>/dev/null || echo "[INFO] $toxiproxy_docker_name was not running"
+        docker run --pull always --detach \
+            --name "$toxiproxy_docker_name" \
+            --hostname "$toxiproxy_docker_name" \
+            --publish 8474:8474 \
+            --publish 55672:55672 \
+            --network "$docker_network_name" \
+            'ghcr.io/shopify/toxiproxy:2.7.0'
+    fi
+}
 
 function start_rabbitmq
 {
+    echo "[INFO] starting RabbitMQ server docker container"
     chmod 0777 "$GITHUB_WORKSPACE/.ci/ubuntu/log"
     docker rm --force "$rabbitmq_docker_name" 2>/dev/null || echo "[INFO] $rabbitmq_docker_name was not running"
-    docker run --pull always --detach --name "$rabbitmq_docker_name" \
+    docker run --pull always --detach \
+        --name "$rabbitmq_docker_name" \
+        --hostname "$rabbitmq_docker_name" \
         --publish 5671:5671 \
         --publish 5672:5672 \
-        --publish 15672:15672 \
+        --network "$docker_network_name" \
         --volume "$GITHUB_WORKSPACE/.ci/ubuntu/enabled_plugins:/etc/rabbitmq/enabled_plugins" \
         --volume "$GITHUB_WORKSPACE/.ci/ubuntu/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro" \
         --volume "$GITHUB_WORKSPACE/.ci/certs:/etc/rabbitmq/certs:ro" \
@@ -109,6 +138,10 @@ function install_ca_certificate
         -pass pass:grapefruit < /dev/null
 }
 
+docker network create "$docker_network_name" || echo "[INFO] network '$docker_network_name' is already created"
+
+start_toxiproxy
+
 start_rabbitmq
 
 wait_rabbitmq

diff --git a/.ci/windows/toxiproxy/LICENSE b/.ci/windows/toxiproxy/LICENSE
@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Shopify
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
diff --git a/.ci/windows/toxiproxy/toxiproxy-cli.exe b/.ci/windows/toxiproxy/toxiproxy-cli.exe
diff --git a/.ci/windows/toxiproxy/toxiproxy-server.exe b/.ci/windows/toxiproxy/toxiproxy-server.exe
diff --git a/.ci/windows/versions.json b/.ci/windows/versions.json
@@ -1,4 +1,4 @@
 {
-  "erlang": "26.2.1",
-  "rabbitmq": "3.12.10"
+  "erlang": "26.2.2",
+  "rabbitmq": "3.12.12"
 }
diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml
@@ -26,7 +26,7 @@ jobs:
       - name: Build (Debug)
         run: dotnet build ${{ github.workspace }}\Build.csproj
       - name: Verify
-        run: dotnet format ${{ github.workspace }}\RabbitMQDotNetClient.sln --no-restore --verify-no-changes --verbosity=diagnostic
+        run: dotnet format ${{ github.workspace }}\RabbitMQDotNetClient.sln --no-restore --verify-no-changes
       - name: APIApproval Test
         run: dotnet test "${{ github.workspace }}\projects\Test\Unit\Unit.csproj" --no-restore --no-build --logger 'console;verbosity=detailed' --filter='FullyQualifiedName=Test.Unit.APIApproval.Approve'
       - name: Unit Tests
@@ -67,7 +67,18 @@ jobs:
         id: install-start-rabbitmq
         run: .\.ci\windows\gha-setup.ps1
       - name: Integration Tests
-        run: dotnet test --environment "RABBITMQ_RABBITMQCTL_PATH=${{ steps.install-start-rabbitmq.outputs.path }}" --environment 'RABBITMQ_LONG_RUNNING_TESTS=true' --environment 'PASSWORD=grapefruit' --environment SSL_CERTS_DIR="${{ github.workspace }}\.ci\certs" "${{ github.workspace }}\projects\Test\Integration\Integration.csproj" --no-restore --no-build --logger 'console;verbosity=detailed'
+        run: |
+          $tx = Start-Job -Verbose -ScriptBlock { & "${{ github.workspace }}\.ci\windows\toxiproxy\toxiproxy-server.exe" }; `
+          Start-Sleep -Seconds 1; `
+          Receive-Job -Job $tx; `
+          & "${{ github.workspace }}\.ci\windows\toxiproxy\toxiproxy-cli.exe" list; `
+          dotnet test `
+            --environment "RABBITMQ_RABBITMQCTL_PATH=${{ steps.install-start-rabbitmq.outputs.path }}" `
+            --environment 'RABBITMQ_LONG_RUNNING_TESTS=true' `
+            --environment 'RABBITMQ_TOXIPROXY_TESTS=true' `
+            --environment 'PASSWORD=grapefruit' `
+            --environment SSL_CERTS_DIR="${{ github.workspace }}\.ci\certs" `
+            "${{ github.workspace }}\projects\Test\Integration\Integration.csproj" --no-restore --no-build --logger 'console;verbosity=detailed'
       - name: Maybe upload RabbitMQ logs
         if: failure()
         uses: actions/upload-artifact@v3
@@ -132,11 +143,11 @@ jobs:
       - name: Build (Debug)
         run: dotnet build ${{ github.workspace }}/Build.csproj
       - name: Verify
-        run: dotnet format ${{ github.workspace }}/RabbitMQDotNetClient.sln --no-restore --verify-no-changes --verbosity=diagnostic
+        run: dotnet format ${{ github.workspace }}/RabbitMQDotNetClient.sln --no-restore --verify-no-changes
       - name: APIApproval Test
         run: dotnet test "${{ github.workspace }}/projects/Test/Unit/Unit.csproj" --no-restore --no-build --logger 'console;verbosity=detailed' --filter='FullyQualifiedName=Test.Unit.APIApproval.Approve'
       - name: Unit Tests
-        run: dotnet test "${{ github.workspace }}/projects/Test/Unit/Unit.csproj" --no-restore --no-build --verbosity=diagnostic --logger 'console;verbosity=detailed'
+        run: dotnet test "${{ github.workspace }}/projects/Test/Unit/Unit.csproj" --no-restore --no-build --logger 'console;verbosity=detailed'
       - name: Upload Build (Debug)
         uses: actions/upload-artifact@v3
         with:
@@ -165,12 +176,13 @@ jobs:
           path: projects
       - name: Start RabbitMQ
         id: start-rabbitmq
-        run: ${{ github.workspace }}/.ci/ubuntu/gha-setup.sh
+        run: ${{ github.workspace }}/.ci/ubuntu/gha-setup.sh toxiproxy
       - name: Integration Tests
         run: |
             dotnet test \
                 --environment "RABBITMQ_RABBITMQCTL_PATH=DOCKER:${{ steps.start-rabbitmq.outputs.id }}" \
                 --environment 'RABBITMQ_LONG_RUNNING_TESTS=true' \
+                --environment 'RABBITMQ_TOXIPROXY_TESTS=true' \
                 --environment 'PASSWORD=grapefruit' \
                 --environment SSL_CERTS_DIR="${{ github.workspace }}/.ci/certs" \
                 "${{ github.workspace }}/projects/Test/Integration/Integration.csproj" --no-restore --no-build --logger 'console;verbosity=detailed'

diff --git a/.gitmodules b/.gitmodules
@@ -2,3 +2,6 @@
 	path = _site
 	url = https://github.com/rabbitmq/rabbitmq-dotnet-client.git
 	branch = gh-pages
+[submodule "projects/toxiproxy-netcore"]
+	path = projects/toxiproxy-netcore
+	url = https://github.com/rabbitmq/toxiproxy-netcore.git
diff --git a/Build.csproj b/Build.csproj
@@ -9,6 +9,7 @@
     <ProjectReference Include="projects/Benchmarks/Benchmarks.csproj" />
     <ProjectReference Include="projects/RabbitMQ.Client/RabbitMQ.Client.csproj" />
     <ProjectReference Include="projects/RabbitMQ.Client.OAuth2/RabbitMQ.Client.OAuth2.csproj" />
+    <ProjectReference Include="projects/toxiproxy-netcore/src/ToxiproxyNetCore/ToxiproxyNetCore.csproj" />
     <ProjectReference Include="projects/Test/Common/Common.csproj" />
     <ProjectReference Include="projects/Test/Applications/CreateChannel/CreateChannel.csproj" />
     <ProjectReference Include="projects/Test/Applications/MassPublish/MassPublish.csproj" />

diff --git a/RabbitMQDotNetClient.sln b/RabbitMQDotNetClient.sln
@@ -38,6 +38,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SequentialIntegration", "pr
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Common", "projects\Test\Common\Common.csproj", "{C11F25F4-7EA1-4874-9E25-DEB42E3A7C67}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ToxiproxyNetCore", "projects\toxiproxy-netcore\src\ToxiproxyNetCore\ToxiproxyNetCore.csproj", "{AB5B7C53-D7EC-4985-A6DE-70178E4B688A}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -84,6 +86,10 @@ Global
 		{C11F25F4-7EA1-4874-9E25-DEB42E3A7C67}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C11F25F4-7EA1-4874-9E25-DEB42E3A7C67}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{C11F25F4-7EA1-4874-9E25-DEB42E3A7C67}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AB5B7C53-D7EC-4985-A6DE-70178E4B688A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AB5B7C53-D7EC-4985-A6DE-70178E4B688A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AB5B7C53-D7EC-4985-A6DE-70178E4B688A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AB5B7C53-D7EC-4985-A6DE-70178E4B688A}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -97,6 +103,7 @@ Global
 		{B01347D8-C327-471B-A1FE-7B86F7684A27} = {EFD4BED5-13A5-4D9C-AADF-CAB7E1573704}
 		{F25725D7-2978-45F4-B90F-25D6F8B71C9E} = {EFD4BED5-13A5-4D9C-AADF-CAB7E1573704}
 		{C11F25F4-7EA1-4874-9E25-DEB42E3A7C67} = {EFD4BED5-13A5-4D9C-AADF-CAB7E1573704}
+		{AB5B7C53-D7EC-4985-A6DE-70178E4B688A} = {EFD4BED5-13A5-4D9C-AADF-CAB7E1573704}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3C6A0C44-FA63-4101-BBF9-2598641167D1}

diff --git a/projects/RabbitMQ.Client/client/api/InternalConstants.cs b/projects/RabbitMQ.Client/client/api/InternalConstants.cs
@@ -37,5 +37,7 @@ internal static class InternalConstants
     {
         internal static readonly TimeSpan DefaultConnectionAbortTimeout = TimeSpan.FromSeconds(5);
         internal static readonly TimeSpan DefaultConnectionCloseTimeout = TimeSpan.FromSeconds(30);
+
+        internal static string Now => DateTime.UtcNow.ToString("s", System.Globalization.CultureInfo.InvariantCulture);
     }
 }
diff --git a/projects/RabbitMQ.Client/client/impl/Connection.Heartbeat.cs b/projects/RabbitMQ.Client/client/impl/Connection.Heartbeat.cs
@@ -103,11 +103,9 @@ private void HeartbeatReadTimerCallback(object? state)
                         _missedHeartbeats++;
                     }
 
-                    // We check against 8 = 2 * 4 because we need to wait for at
-                    // least two complete heartbeat setting intervals before
-                    // complaining, and we've set the socket timeout to a quarter
-                    // of the heartbeat setting in setHeartbeat above.
-                    if (_missedHeartbeats > 2 * 4)
+                    // We need to wait for at least two complete heartbeat setting
+                    // intervals before complaining
+                    if (_missedHeartbeats > 2)
                     {
                         var eose = new EndOfStreamException($"Heartbeat missing with heartbeat == {_heartbeat} seconds");
                         LogCloseError(eose.Message, eose);

diff --git a/projects/RabbitMQ.Client/client/impl/Connection.Receive.cs b/projects/RabbitMQ.Client/client/impl/Connection.Receive.cs
@@ -53,11 +53,6 @@ private async Task MainLoop()
                 await ReceiveLoopAsync(mainLoopToken)
                     .ConfigureAwait(false);
             }
-            catch (OperationCanceledException)
-            {
-                // TODO what to do here?
-                // Debug log?
-            }
             catch (EndOfStreamException eose)
             {
                 // Possible heartbeat exception
@@ -223,7 +218,7 @@ private async Task ClosingLoopAsync(CancellationToken cancellationToken)
         {
             try
             {
-                _frameHandler.ReadTimeout = TimeSpan.Zero;
+                _frameHandler.ReadTimeout = default;
                 // Wait for response/socket closure or timeout
                 await ReceiveLoopAsync(cancellationToken)
                    .ConfigureAwait(false);

diff --git a/projects/Test/Common/IntegrationFixture.cs b/projects/Test/Common/IntegrationFixture.cs
@@ -550,5 +550,7 @@ protected static byte[] GetRandomBody(ushort size = 1024)
 #endif
             return body;
         }
+
+        public static string Now => DateTime.UtcNow.ToString("s", CultureInfo.InvariantCulture);
     }
 }
diff --git a/projects/Test/Integration/Integration.csproj b/projects/Test/Integration/Integration.csproj
@@ -18,6 +18,7 @@
   <ItemGroup>
     <ProjectReference Include="../../RabbitMQ.Client/RabbitMQ.Client.csproj" />
     <ProjectReference Include="../Common/Common.csproj" />
+    <ProjectReference Include="../../toxiproxy-netcore/src/ToxiproxyNetCore/ToxiproxyNetCore.csproj" />
   </ItemGroup>
 
   <!--

diff --git a/projects/Test/Integration/TestConcurrentAccessWithSharedConnection.cs b/projects/Test/Integration/TestConcurrentAccessWithSharedConnection.cs
@@ -33,16 +33,13 @@
 using System.Collections.Generic;
 using System.Threading.Tasks;
 using RabbitMQ.Client;
-using RabbitMQ.Client.Events;
 using Xunit;
 using Xunit.Abstractions;
 
 namespace Test.Integration
 {
     public class TestConcurrentAccessWithSharedConnection : IntegrationFixture
     {
-        private const ushort MessageCount = 256;
-
         public TestConcurrentAccessWithSharedConnection(ITestOutputHelper output)
             : base(output)
         {
@@ -56,30 +53,6 @@ public override async Task InitializeAsync()
             Assert.Null(_channel);
         }
 
-        [Fact]
-        public async Task TestConcurrentChannelOpenAndPublishingWithBlankMessages()
-        {
-            await TestConcurrentChannelOpenAndPublishingWithBodyAsync(Array.Empty<byte>(), 30);
-        }
-
-        [Fact]
-        public async Task TestConcurrentChannelOpenAndPublishingSize64()
-        {
-            await TestConcurrentChannelOpenAndPublishingWithBodyOfSizeAsync(64);
-        }
-
-        [Fact]
-        public async Task TestConcurrentChannelOpenAndPublishingSize256()
-        {
-            await TestConcurrentChannelOpenAndPublishingWithBodyOfSizeAsync(256);
-        }
-
-        [Fact]
-        public Task TestConcurrentChannelOpenAndPublishingSize1024()
-        {
-            return TestConcurrentChannelOpenAndPublishingWithBodyOfSizeAsync(1024);
-        }
-
         [Fact]
         public async Task TestConcurrentChannelOpenCloseLoop()
         {
@@ -92,52 +65,6 @@ await TestConcurrentChannelOperationsAsync(async (conn) =>
             }, 50);
         }
 
-        private Task TestConcurrentChannelOpenAndPublishingWithBodyOfSizeAsync(ushort length, int iterations = 30)
-        {
-            byte[] body = GetRandomBody(length);
-            return TestConcurrentChannelOpenAndPublishingWithBodyAsync(body, iterations);
-        }
-
-        private Task TestConcurrentChannelOpenAndPublishingWithBodyAsync(byte[] body, int iterations)
-        {
-            return TestConcurrentChannelOperationsAsync(async (conn) =>
-            {
-                var tcs = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
-
-                // publishing on a shared channel is not supported
-                // and would missing the point of this test anyway
-                using (IChannel ch = await _conn.CreateChannelAsync())
-                {
-                    await ch.ConfirmSelectAsync();
-
-                    ch.BasicAcks += (object sender, BasicAckEventArgs e) =>
-                    {
-                        if (e.DeliveryTag >= MessageCount)
-                        {
-                            tcs.SetResult(true);
-                        }
-                    };
-
-                    ch.BasicNacks += (object sender, BasicNackEventArgs e) =>
-                    {
-                        tcs.SetResult(false);
-                        Assert.Fail("should never see a nack");
-                    };
-
-                    QueueDeclareOk q = await ch.QueueDeclareAsync(queue: string.Empty, exclusive: true, autoDelete: true);
-                    for (ushort j = 0; j < MessageCount; j++)
-                    {
-                        await ch.BasicPublishAsync("", q.QueueName, body, true);
-                    }
-
-                    Assert.True(await tcs.Task);
-
-                    // NOTE: this is very important before a Dispose();
-                    await ch.CloseAsync();
-                }
-            }, iterations);
-        }
-
         private async Task TestConcurrentChannelOperationsAsync(Func<IConnection, Task> action, int iterations)
         {
             var tasks = new List<Task>();

diff --git a/projects/Test/Integration/TestExchangeDeclare.cs b/projects/Test/Integration/TestExchangeDeclare.cs
@@ -104,7 +104,7 @@ await _channel.ExchangeUnbindAsync(destination: "amq.fanout", source: exchangeNa
         [Fact]
         public async Task TestConcurrentExchangeDeclareAndDelete()
         {
-            var exchangeNames = new List<string>();
+            var exchangeNames = new ConcurrentBag<string>();
             var tasks = new List<Task>();
             NotSupportedException nse = null;
             for (int i = 0; i < 256; i++)

diff --git a/projects/Test/Integration/TestHeartbeats.cs b/projects/Test/Integration/TestHeartbeats.cs
@@ -31,7 +31,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Threading;
 using System.Threading.Tasks;
 using RabbitMQ.Client;
 using Xunit;