test: add visual regression tests with Argos CI

.cspell.json

@@ -31,6 +31,7 @@
     "website/docusaurus.config.localized.json",
     "*.xyz",
     "*.docx",
+    "*.gitignore",
     "versioned_docs",
     "*.min.*",
     "jest/vendor"

.github/workflows/argos.yml

@@ -0,0 +1,39 @@
+name: Argos CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  take-screenshots:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+        with:
+          # A minimal fetch-depth of 2 is required
+          # to automatically detect branch in @argos-ci/cli
+          fetch-depth: 2
+
+      - name: Use Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 16
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps
+
+      - name: Build website fast
+        run: yarn build:website:fast
+
+      - name: Take Argos screenshots
+        run: yarn argos:screenshot
+
+      - name: Upload Argos screenshots
+        continue-on-error: true
+        run: yarn argos:upload

argos/.gitignore

@@ -0,0 +1,5 @@
+
+/screenshots/
+/test-results/
+/playwright-report/
+/playwright/.cache/

argos/README.md

@@ -0,0 +1,21 @@
+# Argos visual regression tests
+
+We use [Argos CI](https://argos-ci.com) to detect visual regressions on Docusaurus.
+
+This workspace can be run manually, but is generally run through the [Argos GitHub Action](../.github/workflows/argos.yml).
+
+The workflow execute those following steps:
+
+- Build the website locally with `yarn build:website:fast`
+- Start the website server with `yarn serve:website` on [http://localhost:3000](http://localhost:3000)
+- Take screenshots of all pages found in `sitemap.xml` with Playwright
+- Upload all screenshots to [Argos CI](https://argos-ci.com)
+
+This workflow runs for `main` and PR branches, and add a commit status to each PR with a visual diff that we can easily inspect.
+
+---
+
+Some additional capabilities:
+
+- Use [./tests/screenshot.spec.ts](./tests/screenshot.spec.ts) to customize the screenshots we take, eventually filter out some useless sitemap pages like versioned docs
+- Use [./tests/screenshot.css](./tests/screenshot.css) to hide flaky CSS elements: iframe, video, gif...

argos/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "argos",
+  "version": "3.0.0-alpha.0",
+  "description": "Argos visual diff tests",
+  "license": "MIT",
+  "private": true,
+  "scripts": {
+    "screenshot": "playwright test",
+    "upload": "npx @argos-ci/cli upload ./screenshots",
+    "report": "playwright show-report"
+  },
+  "dependencies": {
+    "@argos-ci/cli": "^0.4.4",
+    "@argos-ci/playwright": "^0.0.4",
+    "@playwright/test": "^1.30.0",
+    "cheerio": "^1.0.0-rc.12"
+  }
+}

argos/playwright.config.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {devices} from '@playwright/test';
+import type {PlaywrightTestConfig} from '@playwright/test';
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+const config: PlaywrightTestConfig = {
+  testDir: './tests',
+
+  // Run website production built
+  // Need to run "yarn website:build:fast" before
+  webServer: {
+    cwd: '..',
+    port: 3000,
+    command: 'yarn serve:website',
+  },
+
+  // Browsers: only Chrome for now
+  projects: [
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+      },
+    },
+  ],
+};
+
+export default config;

argos/tests/screenshot.css

@@ -0,0 +1,19 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+/*
+Things to hide in Argos screenshots because it's source of flakiness
+ */
+iframe,
+article.yt-lite,
+.theme-last-updated,
+.avatar__photo,
+img[src$='.gif'],
+h2#using-jsx-markup ~ div > div[class*='browserWindowBody']:has(b),
+[class*='playgroundPreview'] {
+  visibility: hidden;
+}

argos/tests/screenshot.spec.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import * as fs from 'fs';
+import {test} from '@playwright/test';
+import {argosScreenshot} from '@argos-ci/playwright';
+import * as cheerio from 'cheerio';
+
+const siteUrl = 'http://localhost:3000';
+const sitemapPath = '../website/build/sitemap.xml';
+const stylesheetPath = './tests/screenshot.css';
+
+// eslint-disable-next-line no-restricted-properties
+const sitemap = fs.readFileSync(sitemapPath).toString();
+// eslint-disable-next-line no-restricted-properties
+const stylesheet = fs.readFileSync(stylesheetPath).toString();
+
+function extractSitemapUrls() {
+  const $ = cheerio.load(sitemap, {xmlMode: true});
+  const urls: string[] = [];
+  $('loc').each(function handleLoc() {
+    urls.push($(this).text());
+  });
+  return urls;
+}
+
+function isBlacklisted(pathname: string) {
+  // Some paths explicitly blacklisted
+  const BlacklistedPathnames: string[] = [
+    '/feature-requests', // Flaky because of Canny widget
+    '/community/canary', // Flaky because of dynamic canary version fetched from npm
+  ];
+
+  return (
+    // changelog docs
+    pathname.startsWith('/changelog') ||
+    // versioned docs
+    pathname.match(/^\/docs\/((\d\.\d\.\d)|(next))\//) ||
+    // manually excluded urls
+    BlacklistedPathnames.includes(pathname)
+  );
+}
+
+function getPathnames(): string[] {
+  const urls = extractSitemapUrls();
+  const pathnamesUnfiltered = urls.map((url) => new URL(url).pathname);
+  const pathnames = pathnamesUnfiltered.filter(
+    (pathname) => !isBlacklisted(pathname),
+  );
+  pathnames.sort();
+  console.log('Pathnames:\n', JSON.stringify(pathnames, null, 2));
+  console.log('Pathnames before filtering', pathnamesUnfiltered.length);
+  console.log('Pathnames after filtering', pathnames.length);
+  return pathnames;
+}
+
+function pathnameToArgosName(pathname: string): string {
+  function removeTrailingSlash(str: string): string {
+    return str.endsWith('/') ? str.slice(0, -1) : str;
+  }
+  function removeLeadingSlash(str: string): string {
+    return str.startsWith('/') ? str.slice(1) : str;
+  }
+
+  pathname = removeTrailingSlash(pathname);
+  pathname = removeLeadingSlash(pathname);
+
+  if (pathname === '') {
+    return 'index';
+  }
+
+  return pathname;
+}
+
+function createPathnameTest(pathname: string) {
+  test(`pathname ${pathname}`, async ({page}) => {
+    const url = siteUrl + pathname;
+    await page.goto(url);
+    await page.addStyleTag({content: stylesheet});
+    // await expect(page).toHaveScreenshot({ fullPage: true, ...options });
+    await argosScreenshot(page, pathnameToArgosName(pathname));
+  });
+}
+
+test.describe('Docusaurus site screenshots', () => {
+  const pathnames = getPathnames();
+  pathnames.forEach(createPathnameTest);
+});

jest.config.mjs

@@ -26,6 +26,7 @@ const ignorePatterns = [
   '/packages/docusaurus-theme-common/lib',
   '/packages/docusaurus-migrate/lib',
   '/jest',
+  '/argos',
 ];
 
 export default {

package.json

@@ -5,6 +5,7 @@
   "private": true,
   "workspaces": [
     "packages/*",
+    "argos",
     "website",
     "test-website-in-workspace",
     "packages/create-docusaurus/templates/*",
@@ -37,6 +38,8 @@
     "crowdin:upload:website": "crowdin upload sources --config ./crowdin-v2.yaml",
     "crowdin:download": "crowdin download --config ./crowdin-v2.yaml",
     "crowdin:download:website": "yarn crowdin:download --language fr --language ko --language pt-BR --language zh-CN --language ja",
+    "argos:screenshot": "yarn workspace argos screenshot",
+    "argos:upload": "yarn workspace argos upload",
     "canary": "yarn canary:bumpVersion && yarn canary:publish",
     "canary:version": "echo 0.0.0-`git rev-list --count HEAD`+`git rev-parse --short HEAD`",
     "canary:bumpVersion": "yarn lerna version `yarn --silent canary:version` --exact --no-push --yes",