VizReg e2e tests: programmatically test all combinations of a given l…

…ist of props/values (#48260)

* Add custom controls decorator

* Add playwright util for parsing e2e controls and testing all combinations via snapshots

* Add HStack e2e story + playwright test

* Add VStack e2e story + playwright test

* Cleanup unused values

* Typo

* Simplify decorator, use globalTypes to enable it

* Update Playwright utils

1. expect prop config as input when computing permutations
2. use new form inputs to submit prop config instead of individual prop controls

* Move list of props to test from Storybook examples to test specs

* Update JSDoc

* Run tests in parallel

* Remove unnecessary imports

* Make the whole VizReg Playwright project to be fully parallel

packages/components/src/h-stack/stories/e2e/index.tsx

@@ -0,0 +1,36 @@
+/**
+ * External dependencies
+ */
+import type { ComponentStory, ComponentMeta } from '@storybook/react';
+
+/**
+ * Internal dependencies
+ */
+import { View } from '../../../view';
+import { HStack } from '../..';
+
+const meta: ComponentMeta< typeof HStack > = {
+	component: HStack,
+	title: 'Components (Experimental)/HStack',
+};
+export default meta;
+
+const Template: ComponentStory< typeof HStack > = ( props ) => {
+	return (
+		<HStack
+			style={ { background: '#eee', minHeight: '3rem' } }
+			{ ...props }
+		>
+			{ [ 'One', 'Two', 'Three', 'Four', 'Five' ].map( ( text ) => (
+				<View key={ text } style={ { background: '#b9f9ff' } }>
+					{ text }
+				</View>
+			) ) }
+		</HStack>
+	);
+};
+
+export const Default: ComponentStory< typeof HStack > = Template.bind( {} );
+Default.args = {
+	spacing: 3,
+};

packages/components/src/v-stack/stories/e2e/index.tsx

@@ -0,0 +1,36 @@
+/**
+ * External dependencies
+ */
+import type { ComponentStory, ComponentMeta } from '@storybook/react';
+
+/**
+ * Internal dependencies
+ */
+import { View } from '../../../view';
+import { VStack } from '../..';
+
+const meta: ComponentMeta< typeof VStack > = {
+	component: VStack,
+	title: 'Components (Experimental)/VStack',
+};
+export default meta;
+
+const Template: ComponentStory< typeof VStack > = ( props ) => {
+	return (
+		<VStack
+			{ ...props }
+			style={ { background: '#eee', minHeight: '3rem' } }
+		>
+			{ [ 'One', 'Two', 'Three', 'Four', 'Five' ].map( ( text ) => (
+				<View key={ text } style={ { background: '#b9f9ff' } }>
+					{ text }
+				</View>
+			) ) }
+		</VStack>
+	);
+};
+
+export const Default: ComponentStory< typeof VStack > = Template.bind( {} );
+Default.args = {
+	spacing: 3,
+};

test/storybook-playwright/playwright.config.ts

@@ -8,6 +8,7 @@ const config: PlaywrightTestConfig = {
 	reporter: [
 		[ 'html', { open: 'on-failure', outputFolder: 'test-results/report' } ],
 	],
+	fullyParallel: true,
 };
 
 export default config;

test/storybook-playwright/specs/hstack.spec.ts

@@ -0,0 +1,55 @@
+/**
+ * External dependencies
+ */
+import { test } from '@playwright/test';
+
+/**
+ * Internal dependencies
+ */
+import {
+	gotoStoryId,
+	getAllPropsPermutations,
+	testSnapshotForPropsConfig,
+} from '../utils';
+
+const PROP_VALUES_TO_TEST = [
+	{
+		propName: 'alignment',
+		valuesToTest: [
+			undefined,
+			'top',
+			'topLeft',
+			'topRight',
+			'left',
+			'center',
+			'right',
+			'bottom',
+			'bottomLeft',
+			'bottomRight',
+			'edge',
+			'stretch',
+		],
+	},
+	{
+		propName: 'direction',
+		valuesToTest: [ undefined, 'row', 'column' ],
+	},
+];
+
+test.describe( 'HStack', () => {
+	test.beforeEach( async ( { page } ) => {
+		await gotoStoryId( page, 'components-experimental-hstack--default', {
+			decorators: { marginChecker: 'show', customE2EControls: 'show' },
+		} );
+	} );
+
+	getAllPropsPermutations( PROP_VALUES_TO_TEST ).forEach( ( propsConfig ) => {
+		test( `should render with ${ JSON.stringify( propsConfig ) }`, async ( {
+			page,
+		} ) => {
+			await page.waitForSelector( '.components-h-stack' );
+
+			await testSnapshotForPropsConfig( page, propsConfig );
+		} );
+	} );
+} );

test/storybook-playwright/specs/vstack.spec.ts

@@ -0,0 +1,55 @@
+/**
+ * External dependencies
+ */
+import { test } from '@playwright/test';
+
+/**
+ * Internal dependencies
+ */
+import {
+	gotoStoryId,
+	getAllPropsPermutations,
+	testSnapshotForPropsConfig,
+} from '../utils';
+
+const PROP_VALUES_TO_TEST = [
+	{
+		propName: 'alignment',
+		valuesToTest: [
+			undefined,
+			'top',
+			'topLeft',
+			'topRight',
+			'left',
+			'center',
+			'right',
+			'bottom',
+			'bottomLeft',
+			'bottomRight',
+			'edge',
+			'stretch',
+		],
+	},
+	{
+		propName: 'direction',
+		valuesToTest: [ undefined, 'row', 'column' ],
+	},
+];
+
+test.describe( 'VStack', () => {
+	test.beforeEach( async ( { page } ) => {
+		await gotoStoryId( page, 'components-experimental-vstack--default', {
+			decorators: { marginChecker: 'show', customE2EControls: 'show' },
+		} );
+	} );
+
+	getAllPropsPermutations( PROP_VALUES_TO_TEST ).forEach( ( propsConfig ) => {
+		test( `should render with ${ JSON.stringify( propsConfig ) }`, async ( {
+			page,
+		} ) => {
+			await page.waitForSelector( '.components-v-stack' );
+
+			await testSnapshotForPropsConfig( page, propsConfig );
+		} );
+	} );
+} );

test/storybook-playwright/storybook/decorators/with-custom-controls.js

@@ -0,0 +1,58 @@
+/**
+ * WordPress dependencies
+ */
+import { useId, useState } from '@wordpress/element';
+
+export const WithCustomControls = ( Story, context ) => {
+	const textareaId = useId();
+	const [ partialProps, setPartialProps ] = useState( {} );
+
+	if ( context.globals.customE2EControls === 'hide' ) {
+		return <Story { ...context } />;
+	}
+
+	const contextWithControlledProps = {
+		...context,
+		// override args with the ones set by custom controls
+		args: { ...context.args, ...partialProps },
+	};
+
+	return (
+		<>
+			<Story { ...contextWithControlledProps } />
+
+			<p>Props:</p>
+			<pre>
+				{ JSON.stringify(
+					contextWithControlledProps.args,
+					undefined,
+					4
+				) }
+			</pre>
+
+			<hr />
+
+			<form
+				name="e2e-controls-form"
+				onSubmit={ ( event ) => {
+					event.preventDefault();
+
+					const propsRawText = event.target.elements.props.value;
+
+					const propsParsed = JSON.parse( propsRawText );
+
+					setPartialProps( ( oldProps ) => ( {
+						...oldProps,
+						...propsParsed,
+					} ) );
+				} }
+			>
+				<p>
+					<label htmlFor={ textareaId }>Raw props</label>
+					<textarea name="props" id={ textareaId } />
+				</p>
+				<button type="submit">Set props</button>
+			</form>
+		</>
+	);
+};

test/storybook-playwright/storybook/preview.js

@@ -1 +1,28 @@
-export * from '../../../storybook/preview';
+/**
+ * Internal dependencies
+ */
+
+import * as basePreviewConfig from '../../../storybook/preview';
+import { WithCustomControls } from './decorators/with-custom-controls';
+
+export const globalTypes = {
+	...basePreviewConfig.globalTypes,
+	customE2EControls: {
+		name: 'Custom E2E Controls',
+		description:
+			'Shows custom UI used by e2e tests for setting props programmatically',
+		defaultValue: 'hide',
+		toolbar: {
+			icon: 'edit',
+			items: [
+				{ value: 'hide', title: 'Hide' },
+				{ value: 'show', title: 'Show' },
+			],
+		},
+	},
+};
+export const decorators = [
+	...basePreviewConfig.decorators,
+	WithCustomControls,
+];
+export const parameters = { ...basePreviewConfig.parameters };

test/storybook-playwright/utils.ts

@@ -2,13 +2,15 @@
  * External dependencies
  */
 import type { Page } from '@playwright/test';
+import { expect } from '@playwright/test';
 
 const STORYBOOK_PORT = '50241';
 
 type Decorators = {
 	css?: 'none' | 'basic' | 'wordpress';
 	direction?: 'ltr' | 'rtl';
 	marginChecker?: 'show' | 'hide';
+	customE2EControls?: 'show' | 'hide';
 };
 type Options = { decorators?: Decorators };
 
@@ -38,3 +40,64 @@ export const gotoStoryId = (
 		{ waitUntil: 'load' }
 	);
 };
+
+/**
+ * Generate all possible permutations of those controls.
+ *
+ * @param propsConfig
+ */
+export const getAllPropsPermutations = (
+	propsConfig: {
+		propName: string;
+		valuesToTest: any[];
+	}[]
+) => {
+	const allPropsPermutations: Record< string, any >[] = [];
+
+	const iterateOverNextPropValues = async (
+		remainingProps: typeof propsConfig,
+		accProps: Record< string, any >
+	) => {
+		const [ propObject, ...restProps ] = remainingProps;
+
+		// Test all values for the given prop.
+		for ( const value of propObject.valuesToTest ) {
+			const newAccProps = {
+				...accProps,
+				[ propObject.propName ]: value,
+			};
+
+			if ( restProps.length === 0 ) {
+				// If we exhausted all of the props to set for this specific combination,
+				// let's add this combination to the `allPropsPermutations` array.
+				allPropsPermutations.push( newAccProps );
+			} else {
+				// If there are more props to iterate through, let's do that through
+				// recursively calling this function.
+				iterateOverNextPropValues( restProps, newAccProps );
+			}
+		}
+	};
+
+	// Start!
+	iterateOverNextPropValues( propsConfig, {} );
+
+	return allPropsPermutations;
+};
+
+export const testSnapshotForPropsConfig = async (
+	page: Page,
+	propsConfig: Record< string, any >
+) => {
+	const textarea = await page.getByLabel( 'Raw props', { exact: true } );
+	const submitButton = await page.getByRole( 'button', {
+		name: 'Set props',
+		exact: true,
+	} );
+
+	await textarea.type( JSON.stringify( propsConfig ) );
+
+	await submitButton.click();
+
+	expect( await page.screenshot() ).toMatchSnapshot();
+};