Merge pull request #812 from wordpress-mobile/update/appium-improvements

e2e test improvements

.circleci/config.yml

@@ -15,36 +15,6 @@ commands:
           key: yarn-v2-{{ arch }}-{{ checksum "yarn.lock" }}
           paths:
             - ~/.cache/yarn
-  build-android:
-    steps:
-      - run:
-          name: Change initial HTML file
-          command: |
-            cp ./bin/example-content/initial-device-tests-html.js ./src/app/initial-html.js
-      - run:
-          name: Bundle Debug android
-          command: yarn bundle:android:test
-      - run:
-          name: Gradle assemble debug android apk
-          command: |
-            cd android
-            ./gradlew clean
-            ./gradlew assembleDebug
-  build-ios:
-    steps:
-    - run:
-        name: Change initial HTML file
-        command: |
-          cp ./bin/example-content/initial-device-tests-html.js ./src/app/initial-html.js
-    - run:
-        name: Bundle iOS
-        command: |
-          yarn bundle:ios:test
-    - run:
-        name: Generate .app file
-        command:  |
-          set +e
-          yarn react-native run-ios --configuration Release
   checkout-gutenberg:
     steps:
       - run:
@@ -92,7 +62,9 @@ jobs:
           command: |
             echo 'export TEST_RN_PLATFORM=android' >> $BASH_ENV
             echo 'export TEST_ENV=sauce' >> $BASH_ENV
-      - build-android
+      - run:
+          name: Bundle Android and Generate debug .apk file for testing
+          command: yarn test:e2e:build-app:android
       - run:
           name: Upload apk to sauce labs
           command: |
@@ -129,18 +101,13 @@ jobs:
         paths:
         - react-native-aztec/ios/Carthage
         - ~/.rncache
-    - build-ios
     - run:
-        name: Zip up .app file
-        command: |
-          cp ./ios/main.jsbundle ./ios/build/Gutenberg/Build/Products/Release-iphonesimulator/gutenberg.app/main.jsbundle
-          cd ./ios/build/Gutenberg/Build/Products/Release-iphonesimulator/
-          zip -r ./Gutenberg.app.zip ./gutenberg.app
-          mv ./Gutenberg.app.zip "$CIRCLE_WORKING_DIRECTORYGutenberg.app.zip"
+        name: Bundle iOS and Generate debug .app file for testing
+        command: yarn test:e2e:build-app:ios
     - run:
         name: Upload .app to sauce labs
         command: |
-          curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg.app.zip?overwrite=true --data-binary @./Gutenberg.app.zip
+          curl -u "$SAUCE_USERNAME:$SAUCE_ACCESS_KEY" -X POST -H "Content-Type: application/octet-stream" https://saucelabs.com/rest/v1/storage/automattic/Gutenberg.app.zip?overwrite=true --data-binary @./ios/Gutenberg.app.zip
     - run:
         name: Run Device Tests
         command: |

.gitignore

@@ -25,6 +25,9 @@ yarn-error.log*
 .gradle/
 android/app/src/main/assets/
 
+# iOS builds
+*.app.zip
+
 # files for the dex VM
 *.dex
 

__device-tests__/blocks/block-interaction.js

@@ -46,14 +46,13 @@ export default class BlockInteraction {
 
 	// Finds the wd element for new block that was added and sets the element attribute
 	// and accessibilityId attributes on this object
-	async setupElement( blockName: string, blocks: Set<string> ) {
+	async setupElement( blockName: string ) {
 		await this.driver.sleep( 2000 );
 		const blockLocator = `block-${ BlockInteraction.index }-${ blockName }`;
 		this.element = await this.driver.elementByAccessibilityId( blockLocator );
 		this.accessibilityId = await this.getAttribute( this.accessibilityIdKey );
 
 		BlockInteraction.index += 1;
-		return blocks;
 	}
 
 	// attempts to type a string to a given element, need for this stems from

__device-tests__/blocks/paragraph-block-interaction.js

@@ -17,9 +17,6 @@ import { isAndroid } from '../helpers/utils';
 import { __ } from '@wordpress/i18n';
 
 export default class ParagraphBlockInteraction extends BlockInteraction {
-	// FLow complaining about type annotation on Set class here but Set<string>(); doesn't resolve
-	// $FlowFixMe
-	static blocks = new Set();
 	textViewElement: wd.PromiseChainWebdriver.Element;
 
 	constructor( driver: wd.PromiseChainWebdriver ) {
@@ -41,7 +38,7 @@ export default class ParagraphBlockInteraction extends BlockInteraction {
 	}
 
 	async setup() {
-		await this.setupElement( this.blockName, ParagraphBlockInteraction.blocks );
+		await this.setupElement( this.blockName );
 		await this.setupTextView();
 	}
 

__device-tests__/gutenberg-editor.test.js

@@ -7,12 +7,11 @@
  */
 import EditorPage from './pages/editor-page';
 import ParagraphBlockInteraction from './blocks/paragraph-block-interaction';
-import { setupAppium, setupDriver, isLocalEnvironment, timer } from './helpers/utils';
+import { setupDriver, isLocalEnvironment, timer, stopDriver } from './helpers/utils';
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 120000;
 
 describe( 'Gutenberg Editor tests', () => {
-	let appium;
 	let driver;
 	let editorPage;
 	let allPassed = true;
@@ -29,46 +28,26 @@ describe( 'Gutenberg Editor tests', () => {
 	}
 
 	beforeAll( async () => {
-		if ( isLocalEnvironment() ) {
-			appium = await setupAppium();
-		}
-
 		driver = await setupDriver();
+		editorPage = new EditorPage( driver );
 	} );
 
 	it( 'should be able to see visual editor', async () => {
-		editorPage = new EditorPage( driver );
-		await editorPage.expect();
+		await expect( editorPage.getBlockList() ).resolves.toBe( true );
 	} );
 
 	it( 'should be able to add a new Paragraph block', async () => {
-		let paragraphBlockInteraction = new ParagraphBlockInteraction( driver );
-		paragraphBlockInteraction = await editorPage.addNewBlock( paragraphBlockInteraction );
+		const paragraphBlockInteraction = new ParagraphBlockInteraction( driver );
+		await editorPage.addNewBlock( paragraphBlockInteraction );
 		await paragraphBlockInteraction.sendText( 'Hello Gutenberg!' );
 		await timer( 3000 );
 		expect( await paragraphBlockInteraction.getText() ).toBe( 'Hello Gutenberg!' );
 	} );
 
 	afterAll( async () => {
-		if ( isLocalEnvironment() ) {
-			if ( driver === undefined ) {
-				if ( appium !== undefined ) {
-					await appium.kill( 'SIGINT' );
-				}
-				return;
-			}
-
-			await driver.quit();
-			await appium.kill( 'SIGINT' );
-		} else {
-			if ( driver === undefined ) {
-				if ( appium !== undefined ) {
-					await appium.kill( 'SIGINT' );
-				}
-				return;
-			}
+		if ( ! isLocalEnvironment() ) {
 			driver.sauceJobStatus( allPassed );
-			await driver.quit();
 		}
+		await stopDriver( driver );
 	} );
 } );

__device-tests__/helpers/appium-local.js

@@ -0,0 +1,48 @@
+/**
+ * @flow
+ * @format
+ * */
+
+/**
+ * External dependencies
+ */
+import childProcess from 'child_process';
+
+// Spawns an appium process
+export const start = ( localAppiumPort: number ) => new Promise < childProcess.ChildProcess > ( ( resolve, reject ) => {
+	const appium = childProcess.spawn( 'appium', [
+		'--port', localAppiumPort.toString(),
+		'--log', './appium-out.log',
+		'--log-no-colors',
+	] );
+
+	let appiumOutputBuffer = '';
+	let resolved = false;
+	appium.stdout.on( 'data', ( data ) => {
+		if ( ! resolved ) {
+			appiumOutputBuffer += data.toString();
+			if ( appiumOutputBuffer.indexOf( 'Appium REST http interface listener started' ) >= 0 ) {
+				resolved = true;
+				resolve( appium );
+			}
+		}
+	} );
+
+	appium.on( 'close', ( code ) => {
+		if ( ! resolved ) {
+			reject( new Error( `Appium process exited with code ${ code }` ) );
+		}
+	} );
+} );
+
+export const stop = async ( appium: ?childProcess.ChildProcess ) => {
+	if ( ! appium ) {
+		return;
+	}
+	await appium.kill( 'SIGINT' );
+};
+
+export default {
+	start,
+	stop,
+};

__device-tests__/helpers/utils.js

@@ -8,16 +8,15 @@
  */
 import childProcess from 'child_process';
 import wd from 'wd';
-import fs from 'fs';
+
 /**
  * Internal dependencies
  */
 import serverConfigs from './serverConfigs';
 import { ios12, android8 } from './caps';
+import AppiumLocal from './appium-local';
 import _ from 'underscore';
 
-const spawn = childProcess.spawn;
-
 // Platform setup
 const defaultPlatform = 'android';
 const rnPlatform = process.env.TEST_RN_PLATFORM || defaultPlatform;
@@ -28,15 +27,15 @@ const testEnvironment = process.env.TEST_ENV || defaultEnvironment;
 
 // Local App Paths
 const defaultAndroidAppPath = './android/app/build/outputs/apk/debug/app-debug.apk';
-const defaultIOSAppPath = './ios/build/gutenberg/Build/Products/Debug-iphonesimulator/gutenberg.app';
+const defaultIOSAppPath = './ios/build/gutenberg/Build/Products/Release-iphonesimulator/gutenberg.app';
 
 const localAndroidAppPath = process.env.ANDROID_APP_PATH || defaultAndroidAppPath;
 const localIOSAppPath = process.env.IOS_APP_PATH || defaultIOSAppPath;
 
 const localAppiumPort = serverConfigs.local.port; // Port to spawn appium process for local runs
+let appiumProcess: ?childProcess.ChildProcess;
 
-// $FlowFixMe
-const timer = ( ms: number ) => new Promise( ( res ) => setTimeout( res, ms ) );
+const timer = ( ms: number ) => new Promise < {} > ( ( res ) => setTimeout( res, ms ) );
 
 const isAndroid = () => {
 	return rnPlatform.toLowerCase() === 'android';
@@ -48,6 +47,16 @@ const isLocalEnvironment = () => {
 
 // Initialises the driver and desired capabilities for appium
 const setupDriver = async () => {
+	if ( isLocalEnvironment() ) {
+		try {
+			appiumProcess = await AppiumLocal.start( localAppiumPort );
+		} catch ( err ) {
+			// Ignore error here, Appium is probably already running (Appium desktop has its own server for instance)
+			// eslint-disable-next-line no-console
+			console.log( 'Could not start Appium server', err.toString() );
+		}
+	}
+
 	const serverConfig = isLocalEnvironment() ? serverConfigs.local : serverConfigs.sauce;
 	const driver = wd.promiseChainRemote( serverConfig );
 
@@ -56,6 +65,19 @@ const setupDriver = async () => {
 		desiredCaps = _.clone( android8 );
 		if ( isLocalEnvironment() ) {
 			desiredCaps.app = localAndroidAppPath;
+			try {
+				const androidVersion = childProcess
+					.execSync( 'adb shell getprop ro.build.version.release' )
+					.toString()
+					.replace( /^\s+|\s+$/g, '' );
+				if ( ! isNaN( androidVersion ) ) {
+					delete desiredCaps.platformVersion;
+					// eslint-disable-next-line no-console
+					console.log( 'Detected Android device running Android %s', androidVersion );
+				}
+			} catch ( error ) {
+				// ignore error
+			}
 		} else {
 			desiredCaps.app = 'sauce-storage:Gutenberg.apk'; // App should be preloaded to sauce storage, this can also be a URL
 		}
@@ -85,25 +107,21 @@ const setupDriver = async () => {
 	return driver;
 };
 
-// Spawns an appium process in the background
-const setupAppium = async () => {
-	const out = fs.openSync( './appium-out.log', 'a' );
-	const err = fs.openSync( './appium-out.log', 'a' );
-
-	const appium = await spawn( 'appium', [ '-p', '' + localAppiumPort ], {
-		detached: true, stdio: [ 'ignore', out, err ],
-
-	} );
+const stopDriver = async ( driver: wd.PromiseChainWebdriver ) => {
+	if ( driver === undefined ) {
+		return;
+	}
+	await driver.quit();
 
-	// Wait a little for server to fire up
-	await timer( 5000 );
-	return appium;
+	if ( appiumProcess !== undefined ) {
+		await AppiumLocal.stop( appiumProcess );
+	}
 };
 
 module.exports = {
 	timer,
-	setupAppium,
 	setupDriver,
 	isLocalEnvironment,
 	isAndroid,
+	stopDriver,
 };

__device-tests__/pages/editor-page.js

@@ -24,9 +24,8 @@ export default class EditorPage {
 		this.driver = driver;
 	}
 
-	async expect( ) {
-		expect( await this.driver.hasElementByAccessibilityId( 'block-list' ) ).toBe( true );
-		return this;
+	async getBlockList() {
+		return this.driver.hasElementByAccessibilityId( 'block-list' );
 	}
 
 	async addNewBlock( block: BlockInteraction ) {