feat(functional): Add new visual regression tests (#846)

* feat(functional): Add visual regression tests using latest deploys

fix: update workflows yml

* tests: update screenshots for CI runs

* feat(functional): Add visual regression tests using latest deploys

* fix: Update tests

* chore: add VR fixes

---------

Co-authored-by: Chancellor Clark <chancellor.clark@bigcommerce.com>

.changeset/unlucky-foxes-hug.md

@@ -0,0 +1,5 @@
+---
+"@bigcommerce/catalyst-core": patch
+---
+
+Migrate visual regression tests

.github/workflows/playwright-ui.yml

@@ -34,9 +34,9 @@ jobs:
       - name: Run Playwright tests
         env:
           PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
-          VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_PROTECTION_BYPASS_CATALYST_LATEST }}
           TEST_ACCOUNT_EMAIL: ${{ secrets.TEST_ACCOUNT_EMAIL }}
           TEST_ACCOUNT_PASSWORD: ${{ secrets.TEST_ACCOUNT_PASSWORD }}
+          VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_PROTECTION_BYPASS_CATALYST_LATEST }}
         run: |
           cd apps/core
           npx playwright test tests/ui/ --project=tests-chromium 

.github/workflows/playwright-visual-regression.yml

@@ -9,7 +9,7 @@ jobs:
     name: Run Visual Regression Tests
     timeout-minutes: 30
     runs-on: macos-14
-    if: ${{ contains(fromJson('["Production – catalyst-storybook", "Preview – catalyst-storybook"]'), github.event.deployment_status.environment) }}
+    if: ${{ contains(fromJson('["Production – catalyst-latest", "Preview – catalyst-latest"]'), github.event.deployment_status.environment) }}
 
     steps:
       - name: Checkout code
@@ -34,10 +34,12 @@ jobs:
       - name: Run Playwright tests
         env:
           PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
+          TEST_ACCOUNT_EMAIL: ${{ secrets.TEST_ACCOUNT_EMAIL }}
+          TEST_ACCOUNT_PASSWORD: ${{ secrets.TEST_ACCOUNT_PASSWORD }}
           VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_PROTECTION_BYPASS_STORYBOOK }}
         run: |
           cd apps/core
-          npx playwright test tests/visual-regression/reactant/components/ --project=tests-chromium
+          npx playwright test tests/visual-regression/components/ --project=tests-chromium
 
       - uses: actions/upload-artifact@v4
         if: failure()

apps/core/components/product-card-carousel/index.tsx

@@ -38,6 +38,7 @@ export const ProductCardCarousel = ({
   showReviews?: boolean;
 }) => {
   const id = useId();
+  const titleId = useId();
 
   if (products.length === 0) {
     return null;
@@ -58,9 +59,9 @@ export const ProductCardCarousel = ({
   }, []);
 
   return (
-    <Carousel aria-labelledby="title" className="mb-14" opts={{ loop: true }}>
+    <Carousel aria-labelledby={titleId} className="mb-14" opts={{ loop: true }}>
       <div className="flex items-center justify-between">
-        <h2 className="text-3xl font-black lg:text-4xl" id="title">
+        <h2 className="text-3xl font-black lg:text-4xl" id={titleId}>
           {title}
         </h2>
         <span className="no-wrap flex">

apps/core/playwright.config.ts

@@ -6,9 +6,11 @@ config();
 export default defineConfig({
   testDir: './tests',
   expect: {
-    toHaveScreenshot: { maxDiffPixels: 100 },
+    toHaveScreenshot: {
+      maxDiffPixelRatio: 0.02,
+    },
   },
-  fullyParallel: true,
+  fullyParallel: !!process.env.CI,
   reporter: 'html',
   use: {
     baseURL: process.env.PLAYWRIGHT_TEST_BASE_URL,

apps/core/tests/routes.ts

@@ -0,0 +1,13 @@
+export default {
+  SHOP_ALL: '/shop-all',
+  SAMPLE_ABLE_BREWING_SYSTEM: '/sample-able-brewing-system',
+  ORBIT_TERRARIUM_LARGE: '/orbit-terrarium-large',
+  BLOG: '/blog',
+  BATH_LUXURY: '/bath/towels/luxury',
+  QUICK_ADD_77: '/shop-all/?showQuickAdd=77',
+  QUICK_ADD_93: '/?showQuickAdd=93',
+  CONTACT_US: '/contact-us',
+  LOGIN: '/login',
+  FOG_LINEN_CHAMBRAY: '/fog-linen-chambray-towel-beige-stripe/',
+  PARFAIT_JAR: '/1-l-le-parfait-jar',
+};

apps/core/tests/visual-regression/components/accordion.spec.ts

@@ -0,0 +1,31 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('accordion expanded', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SHOP_ALL);
+
+  // Act
+  const accordion = page
+    .locator('div[data-state="open"]')
+    .filter({ has: page.getByRole('button', { name: 'Brand', expanded: true }) });
+
+  // Assert
+  await expect(accordion).toHaveScreenshot();
+});
+
+test('accordion closed', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SHOP_ALL);
+
+  // Act
+  await page.getByRole('button', { name: 'Brand', expanded: true }).click();
+
+  const accordion = page
+    .locator('div[data-state="closed"]')
+    .filter({ has: page.getByRole('button', { name: 'Brand', expanded: false }) });
+
+  // Assert
+  await expect(accordion).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/badge.spec.ts

@@ -0,0 +1,24 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('badge with icon', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SAMPLE_ABLE_BREWING_SYSTEM);
+  await page.getByRole('heading', { level: 1, name: '[Sample] Able Brewing System' }).waitFor();
+  await page.getByRole('button', { name: 'Add to Cart' }).click();
+
+  const addToCartNotification = page
+    .getByRole('status')
+    .filter({ hasText: 'Item added to your cart' });
+
+  // Wait for the add to cart notification to appear and disappear
+  await addToCartNotification.waitFor();
+  await addToCartNotification.waitFor({ state: 'detached' });
+
+  // Act
+  const badge = page.getByRole('link', { name: 'Cart Items 1' });
+
+  // Assert
+  await expect(badge).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/blog-post-card.spec.ts

@@ -0,0 +1,15 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('blog post card', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.BLOG);
+  await page.getByRole('heading', { name: 'Blog', exact: true }).waitFor();
+
+  // Act
+  const blogPostCard = page.getByRole('listitem').filter({ hasText: 'Your first blog post!' });
+
+  // Assert
+  await expect(blogPostCard).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/breadcrumbs.spec.ts

@@ -0,0 +1,16 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('breadcrumbs', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.BATH_LUXURY);
+
+  // Act
+  const breadcrumb = page.getByLabel('Breadcrumb');
+
+  await breadcrumb.waitFor();
+
+  // Assert
+  await expect(breadcrumb).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/button.spec.ts

@@ -0,0 +1,42 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('Primary button', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.ORBIT_TERRARIUM_LARGE);
+
+  // Act
+  const button = page.getByRole('button', { name: 'Add to cart' });
+
+  await button.waitFor();
+
+  // Assert
+  await expect(button).toHaveScreenshot();
+});
+
+test('Secondary button', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SHOP_ALL);
+
+  const button = page.getByRole('button', { name: 'Update price' });
+
+  await button.waitFor();
+
+  // Assert
+  await expect(button).toHaveScreenshot();
+});
+
+test('As a child', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SAMPLE_ABLE_BREWING_SYSTEM);
+  await page.getByRole('heading', { level: 1, name: '[Sample] Able Brewing System' }).waitFor();
+
+  // Act
+  await page.getByRole('button', { name: 'Add to Cart' }).first().click();
+  await page.getByRole('link', { name: 'Cart Items 1' }).click();
+  await page.getByText('Shipping cost').waitFor();
+
+  // Assert
+  await expect(page.getByRole('button', { name: 'Add' }).first()).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/carousel.spec.ts

@@ -0,0 +1,13 @@
+import { expect, test } from '@playwright/test';
+
+test('Carousel', async ({ page }) => {
+  // Arrange
+  await page.goto('/');
+  await page.waitForLoadState('networkidle');
+
+  // Act
+  const slides = page.getByRole('region', { name: 'Featured products' });
+
+  // Assert
+  await expect(slides).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/checkbox.spec.ts

@@ -0,0 +1,28 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+test.skip('Disabled checkbox', async () => {});
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+test.skip('Disabled checked checkbox', async () => {});
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+test.skip('Checkbox with error variant', async () => {});
+
+test('Checked checkbox with label', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SHOP_ALL);
+
+  // Act
+  const checkbox = page.getByLabel('Common Good1 products');
+
+  await checkbox.click();
+
+  // Assert
+  await expect(checkbox).toHaveScreenshot();
+});
+
+// eslint-disable-next-line @typescript-eslint/no-empty-function
+test.skip('Checkbox with custom icon', async () => {});

apps/core/tests/visual-regression/components/counter.spec.ts

@@ -0,0 +1,16 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('Counter default', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.QUICK_ADD_77);
+
+  // Act
+  const spinButton = page.getByRole('spinbutton', { name: 'Number' });
+
+  await spinButton.waitFor();
+
+  // Assert
+  await expect(spinButton).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/datepicker.spec.ts

@@ -0,0 +1,16 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('Date picker', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.QUICK_ADD_77);
+
+  // Act
+  const datePicker = page.getByRole('dialog').getByPlaceholder('MM/DD/YYYY');
+
+  await datePicker.waitFor();
+
+  // Assert
+  await expect(datePicker).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/footer.spec.ts

@@ -0,0 +1,14 @@
+import { expect, test } from '@playwright/test';
+
+test('Footer', async ({ page }) => {
+  // Arrange
+  await page.goto('/');
+
+  // Act
+  const footer = page.locator('section').filter({ hasText: 'CategoriesShop' });
+
+  await footer.waitFor();
+
+  // Assert
+  await expect(footer).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/form.spec.ts

@@ -0,0 +1,16 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('Form', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.CONTACT_US);
+
+  // Act
+  const form = page.getByRole('heading', { name: 'Contact Us' });
+
+  await form.waitFor();
+
+  // Assert
+  await expect(form.locator('..').locator('..')).toHaveScreenshot();
+});

apps/core/tests/visual-regression/components/gallery.spec.ts

@@ -0,0 +1,29 @@
+import { expect, test } from '@playwright/test';
+
+import routes from '~/tests/routes';
+
+test('Gallery image', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SAMPLE_ABLE_BREWING_SYSTEM);
+
+  // Act
+  const gallery = page.getByRole('figure').locator('img');
+
+  await gallery.waitFor();
+
+  // Assert
+  await expect(gallery).toHaveScreenshot();
+});
+
+test('Gallery thumbnail image', async ({ page }) => {
+  // Arrange
+  await page.goto(routes.SAMPLE_ABLE_BREWING_SYSTEM);
+
+  // Act
+  const thumbnail = page.getByLabel('Thumbnail navigation');
+
+  await thumbnail.waitFor();
+
+  // Assert
+  await expect(page.getByLabel('Thumbnail navigation')).toHaveScreenshot();
+});