feat: support Playwright --repeat-each option

__fixtures__/screenshots/penelope.png.argos.json

@@ -6,5 +6,7 @@
   "browser": { "name": "chromium", "version": "119.0.6045.9" },
   "automationLibrary": { "name": "playwright", "version": "1.39.0" },
   "sdk": { "name": "@argos-ci/playwright", "version": "0.0.7" },
-  "threshold": 0.2
+  "transient": {
+    "threshold": 0.2
+  }
 }

packages/core/src/upload.test.ts

@@ -27,6 +27,7 @@ describe("#upload", () => {
           hash: expect.stringMatching(/^[A-Fa-f0-9]{64}$/),
           metadata: null,
           threshold: null,
+          baseName: null,
         },
         {
           name: "penelope.png",
@@ -58,6 +59,7 @@ describe("#upload", () => {
             },
           },
           threshold: 0.2,
+          baseName: null,
         },
         {
           name: "nested/alicia.jpg",
@@ -69,6 +71,7 @@ describe("#upload", () => {
           hash: expect.stringMatching(/^[A-Fa-f0-9]{64}$/),
           metadata: null,
           threshold: null,
+          baseName: null,
         },
       ],
     });

packages/core/src/upload.ts

@@ -142,9 +142,11 @@ export async function upload(params: UploadParameters) {
         pwTracePath ? hashFile(pwTracePath) : null,
       ]);
 
-      const threshold = metadata?.threshold ?? null;
+      const threshold = metadata?.transient?.threshold ?? null;
+      const baseName = metadata?.transient?.baseName ?? null;
+
       if (metadata) {
-        delete metadata.threshold;
+        delete metadata.transient;
       }
 
       return {
@@ -153,6 +155,7 @@ export async function upload(params: UploadParameters) {
         optimizedPath,
         metadata,
         threshold,
+        baseName,
         pwTrace:
           pwTracePath && pwTraceHash
             ? { path: pwTracePath, hash: pwTraceHash }
@@ -239,6 +242,7 @@ export async function upload(params: UploadParameters) {
       metadata: screenshot.metadata,
       pwTraceKey: screenshot.pwTrace?.hash ?? null,
       threshold: screenshot.threshold ?? config?.threshold ?? null,
+      baseName: screenshot.baseName,
     })),
     parallel: config.parallel,
     parallelTotal: config.parallelTotal,

packages/cypress/src/support.ts

@@ -161,9 +161,11 @@ Cypress.Commands.add(
           },
         };
 
+        metadata.transient = {};
+
         if (options.threshold !== undefined) {
           validateThreshold(options.threshold);
-          metadata.threshold = options.threshold;
+          metadata.transient.threshold = options.threshold;
         }
 
         cy.writeFile(getMetadataPath(ref.props.path), JSON.stringify(metadata));

packages/playwright/src/index.ts

@@ -115,6 +115,26 @@ async function setup(page: Page, options: ArgosScreenshotOptions) {
   };
 }
 
+/**
+ * Get the screenshot names based on the test info.
+ */
+function getScreenshotNames(name: string, testInfo: TestInfo | null) {
+  if (testInfo) {
+    const projectName = `${testInfo.project.name}/${name}`;
+
+    if (testInfo.repeatEachIndex > 0) {
+      return {
+        name: `${projectName} repeat-${testInfo.repeatEachIndex}`,
+        baseName: projectName,
+      };
+    }
+
+    return { name: projectName, baseName: null };
+  }
+
+  return { name, baseName: null };
+}
+
 /**
  * Stabilize the UI and takes a screenshot of the application under test.
  *
@@ -215,19 +235,24 @@ export async function argosScreenshot(
       ((window as any).__ARGOS__ as ArgosGlobal).waitForStability(),
     );
 
+    const names = getScreenshotNames(name, testInfo);
+
     const metadata = await collectMetadata(testInfo);
+    metadata.transient = {};
+
     if (options.threshold !== undefined) {
       validateThreshold(options.threshold);
-      metadata.threshold = options.threshold;
+      metadata.transient.threshold = options.threshold;
+    }
+
+    if (names.baseName) {
+      metadata.transient.baseName = `${names.baseName}.png`;
     }
-    const nameInProject = testInfo?.project.name
-      ? `${testInfo.project.name}/${name}`
-      : name;
 
     const screenshotPath =
       useArgosReporter && testInfo
-        ? testInfo.outputPath("argos", `${nameInProject}.png`)
-        : resolve(screenshotFolder, `${nameInProject}.png`);
+        ? testInfo.outputPath("argos", `${names.name}.png`)
+        : resolve(screenshotFolder, `${names.name}.png`);
 
     const dir = dirname(screenshotPath);
     if (dir !== screenshotFolder) {
@@ -248,11 +273,11 @@ export async function argosScreenshot(
 
     if (useArgosReporter && testInfo) {
       await Promise.all([
-        testInfo.attach(getAttachmentName(nameInProject, "metadata"), {
+        testInfo.attach(getAttachmentName(names.name, "metadata"), {
           path: getMetadataPath(screenshotPath),
           contentType: "application/json",
         }),
-        testInfo.attach(getAttachmentName(nameInProject, "screenshot"), {
+        testInfo.attach(getAttachmentName(names.name, "screenshot"), {
           path: screenshotPath,
           contentType: "image/png",
         }),
@@ -267,9 +292,7 @@ export async function argosScreenshot(
       const viewportSize = resolveViewport(viewport);
       await page.setViewportSize(viewportSize);
       await stabilizeAndScreenshot(
-        getScreenshotName(name, {
-          viewportWidth: viewportSize.width,
-        }),
+        getScreenshotName(name, { viewportWidth: viewportSize.width }),
       );
     }
 

packages/playwright/src/metadata.ts

@@ -65,6 +65,7 @@ export async function getTestMetadataFromTestInfo(testInfo: TestInfo) {
     titlePath: testInfo.titlePath,
     retry: testInfo.retry,
     retries: testInfo.project.retries,
+    repeat: testInfo.repeatEachIndex,
     location: {
       file: repositoryPath
         ? relative(repositoryPath, testInfo.file)
@@ -86,6 +87,7 @@ export async function getTestMetadataFromTestCase(
     titlePath: testCase.titlePath(),
     retry: testResult.retry,
     retries: testCase.retries,
+    repeat: testCase.repeatEachIndex,
     location: {
       file: repositoryPath
         ? relative(repositoryPath, testCase.location.file)

packages/playwright/src/reporter.ts

@@ -118,6 +118,19 @@ async function getParallelFromConfig(
   };
 }
 
+/**
+ * Get the automatic screenshot name.
+ */
+function getAutomaticScreenshotName(test: TestCase, result: TestResult) {
+  let name = test.titlePath().join(" ");
+  name += result.retry > 0 ? ` #${result.retry + 1}` : "";
+  name +=
+    result.status === "failed" || result.status === "timedOut"
+      ? " (failed)"
+      : "";
+  return name;
+}
+
 class ArgosReporter implements Reporter {
   rootUploadDirectoryPromise: null | Promise<string>;
   uploadDirectoryPromises: Map<string, Promise<string>>;
@@ -162,16 +175,6 @@ class ArgosReporter implements Reporter {
     }
   }
 
-  getAutomaticScreenshotName(test: TestCase, result: TestResult) {
-    let name = test.titlePath().join(" ");
-    name += result.retry > 0 ? ` #${result.retry + 1}` : "";
-    name +=
-      result.status === "failed" || result.status === "timedOut"
-        ? " (failed)"
-        : "";
-    return name;
-  }
-
   /**
    * Get the root upload directory (cached).
    */
@@ -218,7 +221,7 @@ class ArgosReporter implements Reporter {
         // Error screenshots are sent to Argos
         if (checkIsAutomaticScreenshot(attachment)) {
           const metadata = await getMetadataFromTestCase(test, result);
-          const name = this.getAutomaticScreenshotName(test, result);
+          const name = getAutomaticScreenshotName(test, result);
           const path = join(uploadDir, `${name}.png`);
           await Promise.all([
             this.writeFile(path + ".argos.json", JSON.stringify(metadata)),

packages/puppeteer/src/index.ts

@@ -212,9 +212,11 @@ export async function argosScreenshot(
       },
     };
 
+    metadata.transient = {};
+
     if (options?.threshold !== undefined) {
       validateThreshold(options.threshold);
-      metadata.threshold = options.threshold;
+      metadata.transient.threshold = options.threshold;
     }
 
     return metadata;

packages/util/src/metadata-io.test.ts

@@ -28,7 +28,9 @@ describe("#readMetadata", () => {
       browser: { name: "chromium", version: "119.0.6045.9" },
       automationLibrary: { name: "playwright", version: "1.39.0" },
       sdk: { name: "@argos-ci/playwright", version: "0.0.7" },
-      threshold: 0.2,
+      transient: {
+        threshold: 0.2,
+      },
     });
   });
 });

packages/util/src/metadata.ts

@@ -12,6 +12,7 @@ export type ScreenshotMetadata = {
     titlePath: string[];
     retries?: number;
     retry?: number;
+    repeat?: number;
     location?: {
       file: string;
       line: number;
@@ -30,7 +31,11 @@ export type ScreenshotMetadata = {
     name: string;
     version: string;
   };
-  threshold?: number;
+  // Metdata used to pass informations later removed from metadata.
+  transient?: {
+    threshold?: number;
+    baseName?: string;
+  };
 };
 
 /**