Skip to content

Commit

Permalink
test: allow updating test snapshots via env (#2927)
Browse files Browse the repository at this point in the history
  • Loading branch information
@mxschmitt
mxschmitt authored May 1, 2024
1 parent c14ebe5 commit 9ef9165
Show file tree
Hide file tree
Showing 13 changed files with 101 additions and 113 deletions.
1 change: 0 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,6 @@ MigrationBackup/
.idea
src/Playwright.sln.DotSettings
src/Playwright/.drivers
src/Playwright.Tests/Screenshots/**/test.png
src/Playwright/Microsoft.Playwright.xml

testCert.cer
Expand Down
53 changes: 53 additions & 0 deletions src/Playwright.Tests/BaseTests/PlaywrightAssert.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@

using System.Text.Json;
using NUnit.Framework.Constraints;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;

namespace Microsoft.Playwright.Tests;

Expand All @@ -50,4 +52,55 @@ internal static void AreJsonEqual(object expected, object actual)
{
Assert.AreEqual(JsonSerializer.Serialize(expected), JsonSerializer.Serialize(actual));
}

internal static void ToMatchSnapshot(string expectedImageName, string actual)
=> ToMatchSnapshot(expectedImageName, File.ReadAllBytes(actual));

internal static void ToMatchSnapshot(string expectedImageName, byte[] actual)
{
const int pixelThreshold = 10;
const decimal totalTolerance = 0.05m;

var goldenPathDir = Path.Combine(TestUtils.FindParentDirectory("Screenshots"), TestConstants.BrowserName);
var expectedImagePath = Path.Combine(goldenPathDir, expectedImageName);
if (Environment.GetEnvironmentVariable("UPDATE_SNAPSHOTS") == "1")
{
File.WriteAllBytes(expectedImagePath, actual);
}

var expectedImage = Image.Load<Rgb24>(expectedImagePath);
var actualImage = Image.Load<Rgb24>(actual);

if (expectedImage.Width != actualImage.Width || expectedImage.Height != actualImage.Height)
{
Assert.Fail("Expected image dimensions do not match actual image dimensions.\n" +
$"Expected: {expectedImage.Width}x{expectedImage.Height}\n" +
$"Actual: {actualImage.Width}x{actualImage.Height}");
return;
}

int invalidPixelsCount = 0;
for (int y = 0; y < expectedImage.Height; y++)
{
for (int x = 0; x < expectedImage.Width; x++)
{
var pixelA = expectedImage[x, y];
var pixelB = actualImage[x, y];


if (Math.Abs(pixelA.R - pixelB.R) > pixelThreshold ||
Math.Abs(pixelA.G - pixelB.G) > pixelThreshold ||
Math.Abs(pixelA.B - pixelB.B) > pixelThreshold)
{
invalidPixelsCount++;
}
}
}
// cast invalidPixelsCount to decimal to avoid integer division
if (((decimal)invalidPixelsCount / (expectedImage.Height * expectedImage.Width)) > totalTolerance)
{
Assert.Fail($"Expected image to match snapshot but it did not. {invalidPixelsCount} pixels do not match.\n" +
$"Set the environment variable 'UPDATE_SNAPSHOTS' to '1' to update the snapshot.");
}
}
}
26 changes: 13 additions & 13 deletions src/Playwright.Tests/ElementHandleScreenshotTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public async Task ShouldWork()
await Page.EvaluateAsync("window.scrollBy(50, 100)");
var elementHandle = await Page.QuerySelectorAsync(".box:nth-of-type(3)");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should take into account padding and border")]
Expand All @@ -56,7 +56,7 @@ await Page.SetContentAsync(@"
<div id=""d""></div>");
var elementHandle = await Page.QuerySelectorAsync("div#d");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-padding-border.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-padding-border.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should capture full element when larger than viewport in parallel")]
Expand Down Expand Up @@ -84,7 +84,7 @@ await Page.SetContentAsync(@"
var screenshotTasks = elementHandles.Select(e => e.ScreenshotAsync()).ToArray();
await TaskUtils.WhenAll(screenshotTasks);

Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-larger-than-viewport.png", screenshotTasks.ElementAt(2).Result));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-larger-than-viewport.png", screenshotTasks.ElementAt(2).Result);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should capture full element when larger than viewport")]
Expand All @@ -110,7 +110,7 @@ await Page.SetContentAsync(@"

var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-larger-than-viewport.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-larger-than-viewport.png", screenshot);
await TestUtils.VerifyViewportAsync(Page, 500, 500);
}

Expand All @@ -136,7 +136,7 @@ await Page.SetContentAsync(@"
<div class=""to-screenshot""></div>");
var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-scrolled-into-view.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-scrolled-into-view.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should scroll 15000px into view")]
Expand All @@ -161,7 +161,7 @@ await Page.SetContentAsync(@"
<div class=""to-screenshot""></div>");
var elementHandle = await Page.QuerySelectorAsync("div.to-screenshot");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-scrolled-into-view.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-scrolled-into-view.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with a rotated element")]
Expand All @@ -179,7 +179,7 @@ await Page.SetContentAsync(@"
");
var elementHandle = await Page.QuerySelectorAsync("div");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-rotate.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-rotate.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should fail to screenshot a detached element")]
Expand Down Expand Up @@ -221,7 +221,7 @@ public async Task ShouldWaitForVisible()
await elementHandle.EvaluateAsync("e => e.style.visibility = 'visible'");

byte[] screenshot = await task;
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work for an element with fractional dimensions")]
Expand All @@ -230,7 +230,7 @@ public async Task ShouldWorkForAnElementWithFractionalDimensions()
await Page.SetContentAsync("<div style=\"width:48.51px;height:19.8px;border:1px solid black;\"></div>");
var elementHandle = await Page.QuerySelectorAsync("div");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-fractional.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-fractional.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with a mobile viewport")]
Expand All @@ -252,7 +252,7 @@ public async Task ShouldWorkWithAMobileViewport()
var elementHandle = await page.QuerySelectorAsync(".box:nth-of-type(3)");
byte[] screenshot = await elementHandle.ScreenshotAsync();

Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-mobile.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-mobile.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work with device scale factor")]
Expand All @@ -274,7 +274,7 @@ public async Task ShouldWorkWithDeviceScaleFactor()
var elementHandle = await page.QuerySelectorAsync(".box:nth-of-type(3)");
byte[] screenshot = await elementHandle.ScreenshotAsync();

Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-mobile-dsf.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-mobile-dsf.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should work for an element with an offset")]
Expand All @@ -283,7 +283,7 @@ public async Task ShouldWorkForAnElementWithAnOffset()
await Page.SetContentAsync("<div style=\"position:absolute; top: 10.3px; left: 20.4px;width:50.3px;height:20.2px;border:1px solid black;\"></div>");
var elementHandle = await Page.QuerySelectorAsync("div");
byte[] screenshot = await elementHandle.ScreenshotAsync();
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-fractional-offset.png", screenshot));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-fractional-offset.png", screenshot);
}

[PlaywrightTest("elementhandle-screenshot.spec.ts", "should take screenshots when default viewport is null")]
Expand Down Expand Up @@ -399,6 +399,6 @@ public async Task PathOptionShouldCreateSubdirectories()
using var tmpDir = new TempDirectory();
string outputPath = Path.Combine(tmpDir.Path, "these", "are", "directories", "screenshot.png");
await elementHandle.ScreenshotAsync(new() { Path = outputPath });
Assert.True(ScreenshotHelper.PixelMatch("screenshot-element-bounding-box.png", outputPath));
PlaywrightAssert.ToMatchSnapshot("screenshot-element-bounding-box.png", outputPath);
}
}
72 changes: 0 additions & 72 deletions src/Playwright.Tests/Helpers/ScreenshotHelper.cs
View file

This file was deleted.

6 changes: 3 additions & 3 deletions src/Playwright.Tests/PageRequestFulfillTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

namespace Microsoft.Playwright.Tests;

public class RequestFulfillTests : PageTestEx
public class PageRequestFulfillTests : PageTestEx
{
[PlaywrightTest("page-request-fulfill.spec.ts", "should work")]
public async Task ShouldWork()
Expand Down Expand Up @@ -88,7 +88,7 @@ await Page.EvaluateAsync(@"PREFIX => {
return new Promise(fulfill => img.onload = fulfill);
}", Server.Prefix);
var img = await Page.QuerySelectorAsync("img");
Assert.True(ScreenshotHelper.PixelMatch("mock-binary-response.png", await img.ScreenshotAsync()));
PlaywrightAssert.ToMatchSnapshot("mock-binary-response.png", await img.ScreenshotAsync());
}

[PlaywrightTest("page-request-fulfill.spec.ts", "should allow mocking svg with charset")]
Expand Down Expand Up @@ -116,7 +116,7 @@ await Page.EvaluateAsync(@"PREFIX => {
return new Promise(fulfill => img.onload = fulfill);
}", Server.Prefix);
var img = await Page.QuerySelectorAsync("img");
Assert.True(ScreenshotHelper.PixelMatch("mock-binary-response.png", await img.ScreenshotAsync()));
PlaywrightAssert.ToMatchSnapshot("mock-binary-response.png", await img.ScreenshotAsync());
}

[PlaywrightTest("page-request-fulfill.spec.ts", "should stringify intercepted request response headers")]
Expand Down
Loading

0 comments on commit 9ef9165

Please sign in to comment.