Fix flaky CI tests (again 😓 ) (#6780)

* network idle

* Use **/upload

* Test flakes

* Fix pythont tests

* add changeset

* Use right wait

* upload

* trigger ci

* swap order

* networkidle❓

* lint'

* df

* 2 workers

* networkidle

* Fix cli test

* pause

* 3.8 fix

* 3.8

* no promise.all

* trigger ci

* Add code

* mark xfail

* trigger ci

* timeout

* Empty

* Flake

* test

* Flakes

* change test

* awaits

* cleanup

* fixes

---------

Co-authored-by: gradio-pr-bot <gradio-pr-bot@users.noreply.github.com>
Co-authored-by: Abubakar Abid <abubakar@huggingface.co>

.changeset/social-dancers-yawn.md

@@ -0,0 +1,5 @@
+---
+"gradio": patch
+---
+
+fix:Fix flaky CI tests (again 😓 )

.config/playwright.config.js

@@ -1,4 +1,6 @@
-export default {
+import { defineConfig } from "@playwright/test";
+
+export default defineConfig({
 	use: {
 		screenshot: "only-on-failure",
 		trace: "retain-on-failure",
@@ -8,7 +10,10 @@ export default {
 			args: ["--disable-web-security"]
 		}
 	},
+	expect: { timeout: 60000 },
+	timeout: 90000,
 	testMatch: /.*.spec.ts/,
 	testDir: "..",
-	globalSetup: "./playwright-setup.js"
-};
+	globalSetup: "./playwright-setup.js",
+	workers: process.env.CI ? 1 : undefined
+});

.github/workflows/ui.yml

@@ -71,7 +71,7 @@ jobs:
       - name: run browser tests
         run: |
           . venv/bin/activate
-          pnpm test:browser
+          CI=1 pnpm test:browser
       - name: upload screenshots
         uses: actions/upload-artifact@v3
         if: always()

gradio/cli/commands/components/_create_utils.py

@@ -13,8 +13,8 @@
 
 
 def _in_test_dir():
-    """Check if the current working directory ends with gradio/js/gradio-preview/test."""
-    return Path.cwd().parts[-4:] == ("gradio", "js", "gradio-preview", "test")
+    """Check if the current working directory ends with gradio/js/preview/test."""
+    return Path.cwd().parts[-4:] == ("gradio", "js", "preview", "test")
 
 
 default_demo_code = """

gradio/cli/commands/components/build.py

@@ -1,6 +1,5 @@
 import shutil
 import subprocess
-import sys
 from pathlib import Path
 
 import semantic_version
@@ -83,7 +82,7 @@ def _build(
             else:
                 live.update(":white_check_mark: Build succeeded!")
 
-        cmds = [sys.executable, "-m", "build", str(name)]
+        cmds = [shutil.which("python"), "-m", "build", str(name)]
         live.update(f":construction_worker: Building... [grey37]({' '.join(cmds)})[/]")
         pipe = subprocess.run(cmds, capture_output=True, text=True)
         if pipe.returncode != 0:

gradio/cli/commands/components/publish.py

@@ -1,11 +1,9 @@
-from __future__ import annotations
-
 import random
 import re
 import shutil
 import tempfile
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 import semantic_version
 from huggingface_hub import HfApi
@@ -65,13 +63,13 @@ def _ignore(s, names):
     return ignored
 
 
-def _get_version_from_file(dist_file: Path) -> str | None:
+def _get_version_from_file(dist_file: Path) -> Optional[str]:
     match = re.search(r"-(\d+\.\d+\.\d+[a-zA-Z]*\d*)-", dist_file.name)
     if match:
         return match.group(1)
 
 
-def _get_max_version(distribution_files: list[Path]) -> str | None:
+def _get_max_version(distribution_files: List[Path]) -> Optional[str]:
     versions = []
     for p in distribution_files:
         version = _get_version_from_file(p)

js/app/test/audio_component_events.spec.ts

@@ -7,10 +7,7 @@ test("Audio click-to-upload uploads audio successfuly. File downloading works an
 		.getByRole("button", { name: "Drop Audio Here - or - Click to Upload" })
 		.click();
 	const uploader = await page.locator("input[type=file]");
-	await Promise.all([
-		uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]),
-		page.waitForResponse("**/upload?*")
-	]);
+	await uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]);
 
 	await expect(page.getByLabel("# Input Change Events")).toHaveValue("1");
 	await expect(page.getByLabel("# Input Upload Events")).toHaveValue("1");
@@ -21,10 +18,7 @@ test("Audio click-to-upload uploads audio successfuly. File downloading works an
 		.getByRole("button", { name: "Drop Audio Here - or - Click to Upload" })
 		.click();
 
-	await Promise.all([
-		uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]),
-		page.waitForResponse("**/upload?*")
-	]);
+	await uploader.setInputFiles(["../../test/test_files/audio_sample.wav"]);
 
 	await expect(page.getByLabel("# Input Change Events")).toHaveValue("3");
 	await expect(page.getByLabel("# Input Upload Events")).toHaveValue("2");
@@ -38,16 +32,13 @@ test("Audio click-to-upload uploads audio successfuly. File downloading works an
 test("Audio drag-and-drop uploads a file to the server correctly.", async ({
 	page
 }) => {
-	await Promise.all([
-		drag_and_drop_file(
-			page,
-			"input[type=file]",
-			"../../test/test_files/audio_sample.wav",
-			"audio_sample.wav",
-			"audio/wav"
-		),
-		page.waitForResponse("**/upload?*")
-	]);
+	await drag_and_drop_file(
+		page,
+		"input[type=file]",
+		"../../test/test_files/audio_sample.wav",
+		"audio_sample.wav",
+		"audio/wav"
+	);
 	await expect(page.getByLabel("# Input Change Events")).toHaveValue("1");
 	await expect(page.getByLabel("# Input Upload Events")).toHaveValue("1");
 });
@@ -67,30 +58,24 @@ test("Audio drag-and-drop displays a warning when the file is of the wrong mime
 
 test("Play, Pause, and stop events work correctly.", async ({ page }) => {
 	const uploader = await page.locator("input[type=file]");
-	await Promise.all([
-		uploader.setInputFiles(["../../demo/audio_debugger/cantina.wav"]),
-		page.waitForResponse("**/upload?*")
-	]);
+	await uploader.setInputFiles(["../../demo/audio_debugger/cantina.wav"]);
+	const event_triggered = async (label: string) => {
+		const value = await page.getByLabel(label).inputValue();
+		expect(Number(value)).toBeGreaterThan(0);
+	};
 
 	await page
 		.getByTestId("waveform-Output Audio")
 		.getByLabel("Play", { exact: true })
 		.click();
+	await expect(async () => event_triggered("# Output Play Events")).toPass();
+
 	await page.getByTestId("waveform-Output Audio").getByLabel("Pause").click();
+	await expect(async () => event_triggered("# Output Pause Events")).toPass();
+
 	await page
 		.getByTestId("waveform-Output Audio")
 		.getByLabel("Play", { exact: true })
 		.click();
-
-	const event_triggered = async (label: string) => {
-		const value = await page.getByLabel(label).inputValue();
-		expect(Number(value)).toBeGreaterThan(0);
-	};
-
-	// toPass will retry the function until it passes or times out
-	// need this because the stop event is only triggered when the video is done playing
-	// hard to time otherwise
-	await expect(async () => event_triggered("# Output Play Events")).toPass();
-	await expect(async () => event_triggered("# Output Pause Events")).toPass();
 	await expect(async () => event_triggered("# Output Stop Events")).toPass();
 });

js/app/test/chatinterface_streaming_echo.spec.ts

@@ -9,51 +9,30 @@ test("chatinterface works with streaming functions and all buttons behave as exp
 	const clear_button = await page.getByRole("button", { name: "🗑️ Clear" });
 	const textbox = await page.getByPlaceholder("Type a message...");
 
-	let iterations: Promise<any>[] = [];
-	page.on("websocket", (ws) => {
-		iterations.push(
-			ws.waitForEvent("framereceived", {
-				predicate: (event) => {
-					return (
-						JSON.parse(event.payload as string).msg === "process_completed"
-					);
-				}
-			})
-		);
-	});
-
 	await textbox.fill("hello");
 	await submit_button.click();
-	await iterations[0];
 	await expect(textbox).toHaveValue("");
-	await expect.poll(async () => page.locator(".chatbot p").count()).toBe(1);
 	const bot_message_0 = await page.locator(".bot.message").nth(0);
 	await expect(bot_message_0).toContainText("You typed: hello");
 
 	await textbox.fill("hi");
 	await submit_button.click();
-	await iterations[1];
 	await expect(textbox).toHaveValue("");
-	await expect.poll(async () => page.locator(".message.bot").count()).toBe(2);
 	const bot_message_1 = await page.locator(".bot").nth(1);
 	await expect(bot_message_1).toContainText("You typed: hi");
 
 	await retry_button.click();
-	await iterations[2];
 	await expect(textbox).toHaveValue("");
-	await expect(bot_message_1).toContainText("You typed: hi");
+	await expect(page.locator(".bot").nth(1)).toContainText("You typed: hi");
 
 	await undo_button.click();
-	await iterations[3];
 	await expect.poll(async () => page.locator(".message.bot").count()).toBe(1);
 	await expect(textbox).toHaveValue("hi");
 
 	await textbox.fill("salaam");
 	await submit_button.click();
-	await iterations[4];
 	await expect(textbox).toHaveValue("");
-	await expect.poll(async () => page.locator(".bot.message").count()).toBe(2);
-	await expect(bot_message_1).toContainText("You typed: salaam");
+	await expect(page.locator(".bot").nth(1)).toContainText("You typed: salaam");
 
 	await clear_button.click();
 	await expect.poll(async () => page.locator(".bot.message").count()).toBe(0);

js/app/test/image_component_events.spec.ts

@@ -6,37 +6,34 @@ test("Image click-to-upload uploads image successfuly. Clear button dispatches e
 }) => {
 	await page.getByRole("button", { name: "Drop Image Here" }).click();
 	const uploader = await page.locator("input[type=file]");
-	await Promise.all([
-		uploader.setInputFiles(["./test/files/cheetah1.jpg"]),
-		page.waitForResponse("**/upload?*?*")
-	]);
+	const change_counter = await page.getByLabel("# Change Events", {
+		exact: true
+	});
+	const clear_counter = await page.getByLabel("# Clear Events");
+	const upload_counter = await page.getByLabel("# Upload Events");
+	const change_output_counter = await page.getByLabel("# Change Events Output");
 
-	await expect(page.getByLabel("# Change Events").first()).toHaveValue("1");
-	await expect(await page.getByLabel("# Upload Events")).toHaveValue("1");
-	await expect(await page.getByLabel("# Change Events Output")).toHaveValue(
-		"1"
-	);
+	await uploader.setInputFiles("./test/files/cheetah1.jpg");
+
+	await expect(change_counter).toHaveValue("1");
+	await expect(upload_counter).toHaveValue("1");
+	await expect(change_output_counter).toHaveValue("1");
 
 	const downloadPromise = page.waitForEvent("download");
 	await page.getByLabel("Download").click();
 	const download = await downloadPromise;
-	// Automatically convert to png in the backend since PIL is very picky
+	// PIL converts from .jpg to .jpeg
 	await expect(download.suggestedFilename()).toBe("cheetah1.jpeg");
 
 	await page.getByLabel("Remove Image").click();
-	await expect(page.getByLabel("# Clear Events")).toHaveValue("1");
-	await expect(page.getByLabel("# Change Events").first()).toHaveValue("2");
+	await expect(clear_counter).toHaveValue("1");
+	await expect(change_counter).toHaveValue("2");
+	await expect(upload_counter).toHaveValue("1");
 
-	await Promise.all([
-		uploader.setInputFiles(["./test/files/gradio-logo.svg"]),
-		page.waitForResponse("**/upload?*?*")
-	]);
-
-	await expect(page.getByLabel("# Change Events").first()).toHaveValue("3");
-	await expect(await page.getByLabel("# Upload Events")).toHaveValue("2");
-	await expect(await page.getByLabel("# Change Events Output")).toHaveValue(
-		"2"
-	);
+	await uploader.setInputFiles("./test/files/gradio-logo.svg");
+	await expect(change_counter).toHaveValue("3");
+	await expect(upload_counter).toHaveValue("2");
+	await expect(change_output_counter).toHaveValue("2");
 
 	const SVGdownloadPromise = page.waitForEvent("download");
 	await page.getByLabel("Download").click();
@@ -45,16 +42,13 @@ test("Image click-to-upload uploads image successfuly. Clear button dispatches e
 });
 
 test("Image drag-to-upload uploads image successfuly.", async ({ page }) => {
-	await Promise.all([
-		drag_and_drop_file(
-			page,
-			"input[type=file]",
-			"./test/files/cheetah1.jpg",
-			"cheetag1.jpg",
-			"image/*"
-		),
-		page.waitForResponse("**/upload?*")
-	]);
+	await drag_and_drop_file(
+		page,
+		"input[type=file]",
+		"./test/files/cheetah1.jpg",
+		"cheetag1.jpg",
+		"image/*"
+	);
 	await expect(page.getByLabel("# Change Events").first()).toHaveValue("1");
 	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
 });
@@ -71,9 +65,7 @@ test("Image copy from clipboard dispatches upload event.", async ({ page }) => {
 		navigator.clipboard.write([new ClipboardItem({ [blob.type]: blob })]);
 	});
 
-	await page.pause();
 	await page.getByLabel("clipboard-image-toolbar-btn").click();
-	await page.pause();
 	await expect(page.getByLabel("# Change Events").first()).toHaveValue("1");
 	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
 });

js/app/test/model3d_component_events.spec.ts

@@ -7,18 +7,19 @@ test("Model3D click-to-upload uploads file successfuly. Upload and clear events
 		.getByRole("button", { name: "Drop File Here - or - Click to Upload" })
 		.click();
 	const uploader = await page.locator("input[type=file]");
-	await Promise.all([
-		uploader.setInputFiles(["./test/files/face.obj"]),
-		page.waitForResponse("**/upload?*?*")
-	]);
+	const change_counter = await page.getByLabel("# Change Events");
+	const upload_counter = await page.getByLabel("# Upload Events");
+	const clear_counter = await page.getByLabel("# Clear Events");
 
-	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
-	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+	await uploader.setInputFiles("./test/files/face.obj");
+
+	await expect(change_counter).toHaveValue("1");
+	await expect(upload_counter).toHaveValue("1");
 
 	await page.getByLabel("Clear").nth(0).click();
-	await expect(page.getByLabel("# Change Events")).toHaveValue("2");
-	await expect(page.getByLabel("# Clear Events")).toHaveValue("1");
-	await expect(page.getByLabel("Clear Value")).toHaveValue("");
+	await expect(change_counter).toHaveValue("2");
+	await expect(clear_counter).toHaveValue("1");
+	await expect(await page.getByLabel("Clear Value")).toHaveValue("");
 
 	const downloadPromise = page.waitForEvent("download");
 	await page.getByLabel("Download").click();
@@ -35,7 +36,6 @@ test("Model3D drag-and-drop uploads a file to the server correctly.", async ({
 		"./test/files/face.obj",
 		"face.obj"
 	);
-	await page.waitForResponse("**/upload?*");
-	await expect(page.getByLabel("# Change Events")).toHaveValue("1");
-	await expect(page.getByLabel("# Upload Events")).toHaveValue("1");
+	await expect(await page.getByLabel("# Change Events")).toHaveValue("1");
+	await expect(await page.getByLabel("# Upload Events")).toHaveValue("1");
 });