diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0234e4011d..3589606cac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,9 +37,9 @@ jobs: if: matrix.os != 'ubuntu-latest' env: BROWSER: ${{ matrix.browser }} - run: dotnet test ./src/Playwright.Tests/Playwright.Tests.csproj -c Debug -f net8.0 --logger:"console;verbosity=detailed" -- Playwright.Retries=1 + run: dotnet test ./src/Playwright.Tests/Playwright.Tests.csproj -c Debug -f net8.0 --logger:"console;verbosity=detailed" - name: Running tests (Linux) if: matrix.os == 'ubuntu-latest' env: BROWSER: ${{ matrix.browser }} - run: xvfb-run dotnet test ./src/Playwright.Tests/Playwright.Tests.csproj -c Debug -f net8.0 --logger:"console;verbosity=detailed" -- Playwright.Retries=1 + run: xvfb-run dotnet test ./src/Playwright.Tests/Playwright.Tests.csproj -c Debug -f net8.0 --logger:"console;verbosity=detailed" diff --git a/src/Playwright.MSTest/PlaywrightTest.cs b/src/Playwright.MSTest/PlaywrightTest.cs index 3260a2678b..8c7ede3c25 100644 --- a/src/Playwright.MSTest/PlaywrightTest.cs +++ b/src/Playwright.MSTest/PlaywrightTest.cs @@ -76,12 +76,6 @@ public async Task Setup() Playwright.Selectors.SetTestIdAttribute("data-testid"); } - [ClassInitialize(InheritanceBehavior.BeforeEachDerivedClass)] - public static void ClassInitialize(TestContext context) - { - PlaywrightTestMethodAttribute.TestContext = context; - } - [TestCleanup] public async Task Teardown() { diff --git a/src/Playwright.MSTest/PlaywrightTestMethodAttribute.cs b/src/Playwright.MSTest/PlaywrightTestMethodAttribute.cs deleted file mode 100644 index 324b245bcb..0000000000 --- a/src/Playwright.MSTest/PlaywrightTestMethodAttribute.cs +++ /dev/null @@ -1,119 +0,0 @@ -/* - * MIT License - * - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using Microsoft.Playwright.TestAdapter; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using MSTestUnitTesting = Microsoft.VisualStudio.TestTools.UnitTesting; - -[assembly: InternalsVisibleToAttribute("Playwright.TestingHarnessTest, PublicKey=0024000004800000940000000602000000240000525341310004000001000100059a04ca5ca77c9b4eb2addd1afe3f8464b20ee6aefe73b8c23c0e6ca278d1a378b33382e7e18d4aa8300dd22d81f146e528d88368f73a288e5b8157da9710fe6f9fa9911fb786193f983408c5ebae0b1ba5d1d00111af2816f5db55871db03d7536f4a7a6c5152d630c1e1886b1a0fb68ba5e7f64a7f24ac372090889be2ffb")] - -namespace Microsoft.Playwright.MSTest; - -internal class PlaywrightTestMethodAttribute : MSTestUnitTesting.TestMethodAttribute -{ - internal static TestContext TestContext { get; set; } = null!; - - private static int RetryCount - { - get => (int)TestContext.Properties["RetryCount"]; - set => TestContext.Properties["RetryCount"] = value; - } - - public override MSTestUnitTesting.TestResult[] Execute(MSTestUnitTesting.ITestMethod testMethod) - { - MSTestUnitTesting.TestResult[] currentResults = new MSTestUnitTesting.TestResult[] { }; - List failedResults = new(); - RetryCount = 0; - while (true) - { - currentResults = base.Execute(testMethod); - var allPassed = (currentResults ?? new MSTestUnitTesting.TestResult[] { }).All(r => r.Outcome == MSTestUnitTesting.UnitTestOutcome.Passed); - - if (allPassed) - { - break; - } - failedResults.Add(currentResults); - if (RetryCount == PlaywrightSettingsProvider.Retries) - { - break; - } - RetryCount++; - } - if (RetryCount > 0) - { - if (!string.IsNullOrEmpty(currentResults.Last().LogOutput)) - { - currentResults.Last().LogOutput += "\n"; - } - currentResults.Last().LogOutput += GenerateOutputLog(failedResults, testMethod); - } - return currentResults.ToArray(); - } - - private static string GenerateOutputLog(List failedRuns, MSTestUnitTesting.ITestMethod testMethod) - { - var logSeparator = new String('=', 80); - string output = $"{logSeparator}\n"; - output += $"Test was retried {RetryCount} time{(RetryCount > 1 ? "s" : "")}.\n"; - - if (failedRuns.Count > 0) - { - output += $"\nFailing test runs:\n"; - foreach (var (run, retry) in failedRuns.Select((run, i) => (run ?? new MSTestUnitTesting.TestResult[] { }, i))) - { - foreach (var result in run) - { - output += new String('-', 40) + "\n"; - output += $" Test: {testMethod.TestClassName} (retry #{RetryCount})\n"; - output += $" Outcome: {result.Outcome}\n"; - if (result.TestFailureException != null) - { - output += $" Exception: \n{Indent(result.TestFailureException.ToString(), 4)}\n"; - } - if (!string.IsNullOrEmpty(result.LogOutput)) - { - output += $" Standard Output Messages: \n{Indent(result.LogOutput.TrimEnd(), 4)}\n"; - } - if (!string.IsNullOrEmpty(result.LogError)) - { - output += $" Standard Error Messages: \n{Indent(result.LogError.TrimEnd(), 4)}\n"; - } - } - } - } - - output += $"{logSeparator}"; - return output; - } - - private static string Indent(string text, int indent) - { - return text.Split(new[] { "\n" }, StringSplitOptions.None).Select(line => new String(' ', indent) + line).Aggregate((a, b) => a + "\n" + b); - } -} diff --git a/src/Playwright.NUnit/PlaywrightTestAttribute.cs b/src/Playwright.NUnit/PlaywrightTestAttribute.cs deleted file mode 100644 index 8ccc5778ec..0000000000 --- a/src/Playwright.NUnit/PlaywrightTestAttribute.cs +++ /dev/null @@ -1,150 +0,0 @@ -/* - * MIT License - * - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using Microsoft.Playwright.TestAdapter; -using NUnit.Framework.Interfaces; -using NUnit.Framework.Internal; -using NUnit.Framework.Internal.Commands; -using NUnitFrameworkBase = NUnit.Framework; - -[assembly: InternalsVisibleToAttribute("Microsoft.Playwright.Tests, PublicKey=0024000004800000940000000602000000240000525341310004000001000100059a04ca5ca77c9b4eb2addd1afe3f8464b20ee6aefe73b8c23c0e6ca278d1a378b33382e7e18d4aa8300dd22d81f146e528d88368f73a288e5b8157da9710fe6f9fa9911fb786193f983408c5ebae0b1ba5d1d00111af2816f5db55871db03d7536f4a7a6c5152d630c1e1886b1a0fb68ba5e7f64a7f24ac372090889be2ffb")] -[assembly: InternalsVisibleToAttribute("Playwright.TestingHarnessTest, PublicKey=0024000004800000940000000602000000240000525341310004000001000100059a04ca5ca77c9b4eb2addd1afe3f8464b20ee6aefe73b8c23c0e6ca278d1a378b33382e7e18d4aa8300dd22d81f146e528d88368f73a288e5b8157da9710fe6f9fa9911fb786193f983408c5ebae0b1ba5d1d00111af2816f5db55871db03d7536f4a7a6c5152d630c1e1886b1a0fb68ba5e7f64a7f24ac372090889be2ffb")] - -namespace Microsoft.Playwright.NUnit; - -[AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = true)] -internal class PlaywrightTestAttribute : NUnitFrameworkBase.TestAttribute, IWrapTestMethod -{ - public PlaywrightTestAttribute() : base() - { - } - - public TestCommand Wrap(TestCommand command) => new RetryTestCommand(command); - - internal class RetryTestCommand : DelegatingTestCommand - { - public RetryTestCommand(TestCommand innerCommand) : base(innerCommand) - { - } - - public override TestResult Execute(TestExecutionContext context) - { - List failedResults = new List(); - context.CurrentTest.Properties.Set("RetryCount", 0); - while (true) - { - try - { - context.CurrentResult = innerCommand.Execute(context); - } - catch (Exception ex) - { - context.CurrentResult = context.CurrentTest.MakeTestResult(); - context.CurrentResult.RecordException(ex); - } - if (context.CurrentResult.ResultState == ResultState.Success) - { - break; - } - failedResults.Add(new SlimTestResult(context.CurrentResult)); - if (Test2RetryCount(context.CurrentTest) == PlaywrightSettingsProvider.Retries) - { - break; - } - context.CurrentTest.Properties.Set("RetryCount", Test2RetryCount(context.CurrentTest) + 1); - } - if (Test2RetryCount(context.CurrentTest) > 0) - { - if (!string.IsNullOrEmpty(context.CurrentResult.Output)) - { - context.CurrentResult.OutWriter.Write("\n"); - } - var retryCount = Test2RetryCount(context.CurrentTest); - context.CurrentResult.OutWriter.Write(GenerateOutputLog(failedResults, retryCount)); - } - return context.CurrentResult; - } - - private static int Test2RetryCount(Test test) - => (int)test.Properties.Get("RetryCount")!; - - private static string GenerateOutputLog(List failedResults, int retryCount) - { - var logSeparator = new String('=', 80); - string output = $"{logSeparator}\n"; - output += $"Test was retried {retryCount} time{(retryCount > 1 ? "s" : "")}.\n"; - - if (failedResults.Count > 0) - { - output += $"\nFailing test runs:\n"; - foreach (var result in failedResults) - { - if (result == null) - { - continue; - } - output += new String('-', 40) + "\n"; - output += $" Test: {result.FullName}\n"; - output += $" Outcome: {result.ResultState}\n"; - if (!string.IsNullOrEmpty(result.Output)) - { - output += $" Standard Output Messages: \n{Indent(result.Output.TrimEnd(), 4)}\n"; - } - if (!string.IsNullOrEmpty(result.Message)) - { - output += $" Message \n{Indent(result.Message!.TrimEnd(), 4)}\n"; - } - } - } - - output += logSeparator; - return output; - } - - private static string Indent(string text, int indent) - { - return text.Split(new[] { "\n" }, StringSplitOptions.None).Select(line => new String(' ', indent) + line).Aggregate((a, b) => a + "\n" + b); - } - } - - private class SlimTestResult - { - public string FullName { get; } - public string Output { get; } - public ResultState ResultState { get; } - public string? Message { get; } - - internal SlimTestResult(TestResult result) - { - FullName = result.FullName; - Output = result.Output; - ResultState = result.ResultState; - Message = result.Message; - } - } -} diff --git a/src/Playwright.TestAdapter/PlaywrightSettingsProvider.cs b/src/Playwright.TestAdapter/PlaywrightSettingsProvider.cs index 9bc5d7d2e3..74dfb65560 100644 --- a/src/Playwright.TestAdapter/PlaywrightSettingsProvider.cs +++ b/src/Playwright.TestAdapter/PlaywrightSettingsProvider.cs @@ -71,11 +71,6 @@ public static float? ExpectTimeout } } - public static int Retries - { - get => _settings?.Retries ?? 0; - } - public static BrowserTypeLaunchOptions LaunchOptions { get diff --git a/src/Playwright.TestingHarnessTest/tests/mstest/retry.spec.ts b/src/Playwright.TestingHarnessTest/tests/mstest/retry.spec.ts deleted file mode 100644 index 49ebf03b32..0000000000 --- a/src/Playwright.TestingHarnessTest/tests/mstest/retry.spec.ts +++ /dev/null @@ -1,188 +0,0 @@ -/* - * MIT License - * - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -import { test, expect } from '../baseTest'; - -test('should not retry a passed test with retries: 0', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.MSTest; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Playwright.TestingHarnessTest.MSTest; - - [TestClass] - public class : PageTest - { - [PlaywrightTestMethod] - public void MyTest() - { - Console.WriteLine("i-was-running"); - } - }`, - }, 'dotnet test'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stdout.match(/i-was-running/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); - -test('should not retry a failed test with retries: 0', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.MSTest; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Playwright.TestingHarnessTest.MSTest; - - [TestClass] - public class : PageTest - { - [PlaywrightTestMethod] - public void MyTest() - { - Console.WriteLine("i-was-running"); - throw new Exception("i-was-broken"); - } - }`, - }, 'dotnet test'); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); - expect(result.total).toBe(1); - expect(result.stdout.match(/i-was-running/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Failed MyTest \[/); -}); - -test('should not retry a passed Mytest with retries: 1', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.MSTest; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Playwright.TestingHarnessTest.MSTest; - - [TestClass] - public class : PageTest - { - [PlaywrightTestMethod] - public void MyTest() - { - Console.WriteLine("i-was-running"); - } - }`, - '.runsettings': ` - - - - 1 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stdout.match(/i-was-running/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); - -test('should retry a failed test with retries: 1', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.MSTest; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Playwright.TestingHarnessTest.MSTest; - - [TestClass] - public class : PageTest - { - [PlaywrightTestMethod] - public void MyTest() - { - Console.Error.WriteLine("i-was-running"); - throw new Exception("i-was-broken"); - } - }`, - '.runsettings': ` - - - - 1 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); - expect(result.total).toBe(1); - expect(result.stdout).toContain('Test was retried 1 time.'); - expect(result.rawStdout).toMatch(/Failed MyTest \[/); -}); - -test('should retry a failed test and stop once it passed', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.MSTest; - using Microsoft.VisualStudio.TestTools.UnitTesting; - - namespace Playwright.TestingHarnessTest.MSTest; - - [TestClass] - public class : PageTest - { - static int retries = 0; - - [PlaywrightTestMethod] - public void MyTest() - { - Console.Error.WriteLine("i-was-running" + retries); - retries++; - if (retries < 5) - throw new Exception("i-was-broken"); - } - }`, - '.runsettings': ` - - - - 10 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stdout).toContain('Test was retried 4 times.'); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); diff --git a/src/Playwright.TestingHarnessTest/tests/nunit/retry.spec.ts b/src/Playwright.TestingHarnessTest/tests/nunit/retry.spec.ts deleted file mode 100644 index 539a4e528e..0000000000 --- a/src/Playwright.TestingHarnessTest/tests/nunit/retry.spec.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - * MIT License - * - * Copyright (c) Microsoft Corporation. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -import { test, expect } from '../baseTest'; - -test('should not retry a passed test with retries: 0', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.NUnit; - using NUnit.Framework; - - namespace Playwright.TestingHarnessTest.NUnit; - - [TestFixture] - public class : PageTest - { - [PlaywrightTest] - public void MyTest() - { - Console.WriteLine("i-was-running"); - } - }`, - }, 'dotnet test'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stdout.match(/i-was-running/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); - -test('should not retry a failed test with retries: 0', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.NUnit; - using NUnit.Framework; - - namespace Playwright.TestingHarnessTest.NUnit; - - [TestFixture] - public class : PageTest - { - [PlaywrightTest] - public void MyTest() - { - Console.WriteLine("i-was-running"); - throw new Exception("i-was-broken"); - } - }`, - }, 'dotnet test'); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); - expect(result.total).toBe(1); - expect(result.rawStdout).toContain("i-was-broken") - expect(result.rawStdout.match(/i-was-broken/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Failed MyTest \[/); -}); - -test('should not retry a passed test with retries: 1', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.NUnit; - using NUnit.Framework; - - namespace Playwright.TestingHarnessTest.NUnit; - - [TestFixture] - public class : PageTest - { - [PlaywrightTest] - public void MyTest() - { - Console.WriteLine("i-was-running"); - } - }`, - '.runsettings': ` - - - - 1 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stdout.match(/i-was-running/g).length).toBe(1); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); - -test('should retry a failed test with retries: 1', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.NUnit; - using NUnit.Framework; - - namespace Playwright.TestingHarnessTest.NUnit; - - [TestFixture] - public class : PageTest - { - [PlaywrightTest] - public void MyTest() - { - Console.Error.WriteLine("i-was-running"); - throw new Exception("i-was-broken"); - } - }`, - '.runsettings': ` - - - - 1 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expect(result.failed).toBe(1); - expect(result.total).toBe(1); - expect(result.rawStdout).toContain("i-was-broken") - expect(result.stderr.match(/i-was-running/g).length).toBe(2); - expect(result.stdout).toContain('Test was retried 1 time.'); - expect(result.rawStdout).toMatch(/Failed MyTest \[/); -}); - -test('should retry a failed test and stop once it passed', async ({ runTest }) => { - const result = await runTest({ - 'ExampleTests.cs': ` - using System; - using Microsoft.Playwright.NUnit; - using NUnit.Framework; - - namespace Playwright.TestingHarnessTest.NUnit; - - [TestFixture] - public class : PageTest - { - static int retries = 0; - - [PlaywrightTest] - public void MyTest() - { - Console.Error.WriteLine("i-was-running"); - retries++; - if (retries < 5) - throw new Exception("i-was-broken"); - } - }`, - '.runsettings': ` - - - - 10 - - `, - }, 'dotnet test --settings=.runsettings'); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.failed).toBe(0); - expect(result.total).toBe(1); - expect(result.stderr.match(/i-was-running/g).length).toBe(5); - expect(result.stdout).toContain('Test was retried 4 times.'); - expect(result.rawStdout).toMatch(/Passed MyTest \[/); -}); diff --git a/src/Playwright.Tests/Attributes/PlaywrightTestAttribute.cs b/src/Playwright.Tests/Attributes/PlaywrightTestAttribute.cs index 45d7e4cd70..7edd173619 100644 --- a/src/Playwright.Tests/Attributes/PlaywrightTestAttribute.cs +++ b/src/Playwright.Tests/Attributes/PlaywrightTestAttribute.cs @@ -94,7 +94,7 @@ public TestCommand Wrap(TestCommand command) /// /// Helper to detect UnobservedTaskExceptions /// - private sealed class UnobservedTaskExceptionCommand : NUnit.PlaywrightTestAttribute.RetryTestCommand + private sealed class UnobservedTaskExceptionCommand : DelegatingTestCommand { public UnobservedTaskExceptionCommand(TestCommand innerCommand) : base(innerCommand) @@ -109,7 +109,7 @@ public override TestResult Execute(TestExecutionContext context) TestResult result = null; try { - result = base.Execute(context); + result = innerCommand.Execute(context); } finally { diff --git a/src/Playwright.Tests/PageClickTests.cs b/src/Playwright.Tests/PageClickTests.cs index d7ba2d9791..e192e2f3ef 100644 --- a/src/Playwright.Tests/PageClickTests.cs +++ b/src/Playwright.Tests/PageClickTests.cs @@ -528,7 +528,8 @@ await Page.EvalOnSelectorAsync("button", @"button => { button.style.display = 'block'; document.body.style.margin = '0'; }"); - + // rafraf for Firefox to kick in the animation. + await Page.EvaluateAsync("() => new Promise(f => requestAnimationFrame(() => requestAnimationFrame(f)))"); await Page.ClickAsync("button"); Assert.AreEqual("Clicked", await Page.EvaluateAsync("window.result")); Assert.AreEqual(300, await Page.EvaluateAsync("pageX")); diff --git a/src/Playwright.Tests/PageClickTimeout4Tests.cs b/src/Playwright.Tests/PageClickTimeout4Tests.cs index 9abfec5ae3..b0270fbcd7 100644 --- a/src/Playwright.Tests/PageClickTimeout4Tests.cs +++ b/src/Playwright.Tests/PageClickTimeout4Tests.cs @@ -35,6 +35,8 @@ await Page.EvalOnSelectorAsync("button", @"button => { button.style.marginLeft = '200px'; }"); + // rafraf for Firefox to kick in the animation. + await Page.EvaluateAsync("() => new Promise(f => requestAnimationFrame(() => requestAnimationFrame(f)))"); var exception = await PlaywrightAssert.ThrowsAsync(() => Page.ClickAsync("button", new() { Timeout = 3000 }));