diff --git a/.github/actions/composite/buildAndroidAPK/action.yml b/.github/actions/composite/buildAndroidAPK/action.yml index 819234df0bc3..fc280ab2a223 100644 --- a/.github/actions/composite/buildAndroidAPK/action.yml +++ b/.github/actions/composite/buildAndroidAPK/action.yml @@ -13,7 +13,7 @@ runs: - uses: ruby/setup-ruby@eae47962baca661befdfd24e4d6c34ade04858f7 with: - ruby-version: '2.7' + ruby-version: "2.7" bundler-cache: true - uses: gradle/gradle-build-action@3fbe033aaae657f011f88f29be9e65ed26bd29ef @@ -26,4 +26,4 @@ runs: uses: actions/upload-artifact@65d862660abb392b8c4a3d1195a2108db131dd05 with: name: ${{ inputs.ARTIFACT_NAME }} - path: android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk + path: android/app/build/outputs/apk/e2e/release/app-e2e-release.apk diff --git a/.github/actions/javascript/getPullRequestDetails/action.yml b/.github/actions/javascript/getPullRequestDetails/action.yml index a59cf55bdf9f..ed2c60f018a1 100644 --- a/.github/actions/javascript/getPullRequestDetails/action.yml +++ b/.github/actions/javascript/getPullRequestDetails/action.yml @@ -13,8 +13,14 @@ inputs: outputs: MERGE_COMMIT_SHA: description: 'The merge_commit_sha of the given pull request' + HEAD_COMMIT_SHA: + description: 'The head_commit_sha of the given pull request' MERGE_ACTOR: description: 'The actor who merged the pull request' + IS_MERGED: + description: 'True if the pull request is merged' + FORKED_REPO_URL: + description: 'Output forked repo URL if PR includes changes from a fork' runs: using: 'node16' main: './index.js' diff --git a/.github/workflows/e2ePerformanceTests.yml b/.github/workflows/e2ePerformanceTests.yml index d8f9cad138d9..f7f1e5fc7ac7 100644 --- a/.github/workflows/e2ePerformanceTests.yml +++ b/.github/workflows/e2ePerformanceTests.yml @@ -84,12 +84,7 @@ jobs: - name: Unmerged PR - Fetch head ref of unmerged PR if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} run: | - if [[ ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} != '' ]]; then - git remote add pr_remote ${{ steps.getPullRequestDetails.outputs.FORKED_REPO_URL }} - git fetch pr_remote ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - else - git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - fi + git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 - name: Unmerged PR - Set dummy git credentials before merging if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} @@ -101,7 +96,7 @@ jobs: if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} id: getMergeCommitShaIfUnmergedPR run: | - git merge --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} + git merge --allow-unrelated-histories --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} env: GITHUB_TOKEN: ${{ github.token }} @@ -140,18 +135,19 @@ jobs: name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }} path: zip - # The downloaded artifact will be a file named "app-e2eRelease.apk" so we have to rename it + # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it - name: Rename baseline APK - run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk" + run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-baseline.apk" - name: Download delta APK uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b + id: downloadDeltaAPK with: name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }} path: zip - name: Rename delta APK - run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-compare.apk" + run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-compare.apk" - name: Copy e2e code into zip folder run: cp -r tests/e2e zip diff --git a/android/app/build.gradle b/android/app/build.gradle index bb53e940563e..1cc6170e69f5 100644 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -58,7 +58,7 @@ project.ext.envConfigFiles = [ adhocRelease: ".env.adhoc", developmentRelease: ".env", developmentDebug: ".env", - e2eRelease: ".env.production" + e2eRelease: "tests/e2e/.env.e2e" ] /** @@ -136,10 +136,20 @@ android { signingConfig signingConfigs.debug } release { - signingConfig signingConfigs.release productFlavors.production.signingConfig signingConfigs.release minifyEnabled enableProguardInReleaseBuilds proguardFiles getDefaultProguardFile("proguard-android.txt"), "proguard-rules.pro" + + signingConfig null + // buildTypes take precedence over productFlavors when it comes to the signing configuration, + // thus we need to manually set the signing config, so that the e2e uses the debug config again. + // In other words, the signingConfig setting above will be ignored when we build the flavor in release mode. + productFlavors.all { flavor -> + // All release builds should be signed with the release config ... + flavor.signingConfig signingConfigs.release + } + // ... except for the e2e flavor, which we maybe want to build locally: + productFlavors.e2e.signingConfig signingConfigs.debug } } diff --git a/fastlane/Fastfile b/fastlane/Fastfile index c7d0f2f4f0f5..dac53193fdc6 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -17,7 +17,7 @@ platform :android do desc "Generate a new local APK for e2e testing" lane :build_e2e do ENV["ENVFILE"]="tests/e2e/.env.e2e" - ENV["ENTRY_FILE"]="#{Dir.pwd}/../src/libs/E2E/reactNativeLaunchingTest.js" + ENV["ENTRY_FILE"]="src/libs/E2E/reactNativeLaunchingTest.js" ENV["E2E_TESTING"]="true" gradle( diff --git a/scripts/android-repackage-app-bundle-and-sign.sh b/scripts/android-repackage-app-bundle-and-sign.sh index fe4ee1e4b8fc..1636edc21388 100755 --- a/scripts/android-repackage-app-bundle-and-sign.sh +++ b/scripts/android-repackage-app-bundle-and-sign.sh @@ -1,4 +1,5 @@ #!/bin/bash +source ./scripts/shellUtils.sh ### # Takes an android app that has been built with the debug keystore, @@ -41,7 +42,7 @@ if [ ! -f "$NEW_BUNDLE_FILE" ]; then echo "Bundle file not found: $NEW_BUNDLE_FILE" exit 1 fi -OUTPUT_APK=$(realpath "$OUTPUT_APK") +OUTPUT_APK=$(get_abs_path "$OUTPUT_APK") # check if "apktool" command is available if ! command -v apktool &> /dev/null then diff --git a/scripts/shellUtils.sh b/scripts/shellUtils.sh index 876933af9766..4c9e2febc34d 100644 --- a/scripts/shellUtils.sh +++ b/scripts/shellUtils.sh @@ -41,3 +41,46 @@ function join_by_string { shift printf "%s" "$first" "${@/#/$separator}" } + +# Usage: get_abs_path +# Will make a path absolute, resolving any relative paths +# example: get_abs_path "./foo/bar" +get_abs_path() { + local the_path=$1 + local -a path_elements + IFS='/' read -ra path_elements <<< "$the_path" + + # If the path is already absolute, start with an empty string. + # We'll prepend the / later when reconstructing the path. + if [[ "$the_path" = /* ]]; then + abs_path="" + else + abs_path="$(pwd)" + fi + + # Handle each path element + for element in "${path_elements[@]}"; do + if [ "$element" = "." ] || [ -z "$element" ]; then + continue + elif [ "$element" = ".." ]; then + # Remove the last element from abs_path + abs_path=$(dirname "$abs_path") + else + # Append element to the absolute path + abs_path="${abs_path}/${element}" + fi + done + + # Remove any trailing '/' + while [[ $abs_path == */ ]]; do + abs_path=${abs_path%/} + done + + # Special case for root + [ -z "$abs_path" ] && abs_path="/" + + # Special case to remove any starting '//' when the input path was absolute + abs_path=${abs_path/#\/\//\/} + + echo "$abs_path" +} \ No newline at end of file diff --git a/src/libs/E2E/API.mock.js b/src/libs/E2E/API.mock.js index 501108025979..47f445f72222 100644 --- a/src/libs/E2E/API.mock.js +++ b/src/libs/E2E/API.mock.js @@ -19,6 +19,7 @@ const mocks = { BeginSignIn: mockBeginSignin, SigninUser: mockSigninUser, OpenApp: mockOpenApp, + ReconnectApp: mockOpenApp, OpenReport: mockOpenReport, AuthenticatePusher: mockAuthenticatePusher, }; diff --git a/src/libs/E2E/reactNativeLaunchingTest.js b/src/libs/E2E/reactNativeLaunchingTest.js index 869f5d1f1f1a..13183c1044db 100644 --- a/src/libs/E2E/reactNativeLaunchingTest.js +++ b/src/libs/E2E/reactNativeLaunchingTest.js @@ -6,11 +6,7 @@ */ import Performance from '../Performance'; - -// start the usual app -Performance.markStart('regularAppStart'); -import '../../../index'; -Performance.markEnd('regularAppStart'); +import * as Metrics from '../Metrics'; import E2EConfig from '../../../tests/e2e/config'; import E2EClient from './client'; @@ -19,6 +15,11 @@ console.debug('=========================='); console.debug('==== Running e2e test ===='); console.debug('=========================='); +// Check if the performance module is available +if (!Metrics.canCapturePerformanceMetrics()) { + throw new Error('Performance module not available! Please set CAPTURE_METRICS=true in your environment file!'); +} + // import your test here, define its name and config first in e2e/config.js const tests = { [E2EConfig.TEST_NAMES.AppStartTime]: require('./tests/appStartTimeTest.e2e').default, @@ -36,20 +37,33 @@ const appReady = new Promise((resolve) => { }); }); -E2EClient.getTestConfig().then((config) => { - const test = tests[config.name]; - if (!test) { - // instead of throwing, report the error to the server, which is better for DX - return E2EClient.submitTestResults({ - name: config.name, - error: `Test '${config.name}' not found`, - }); - } - console.debug(`[E2E] Configured for test ${config.name}. Waiting for app to become ready`); - - appReady.then(() => { - console.debug('[E2E] App is ready, running test…'); - Performance.measureFailSafe('appStartedToReady', 'regularAppStart'); - test(); +E2EClient.getTestConfig() + .then((config) => { + const test = tests[config.name]; + if (!test) { + // instead of throwing, report the error to the server, which is better for DX + return E2EClient.submitTestResults({ + name: config.name, + error: `Test '${config.name}' not found`, + }); + } + + console.debug(`[E2E] Configured for test ${config.name}. Waiting for app to become ready`); + appReady + .then(() => { + console.debug('[E2E] App is ready, running test…'); + Performance.measureFailSafe('appStartedToReady', 'regularAppStart'); + test(); + }) + .catch((error) => { + console.error('[E2E] Error while waiting for app to become ready', error); + }); + }) + .catch((error) => { + console.error("[E2E] Error while running test. Couldn't get test config!", error); }); -}); + +// start the usual app +Performance.markStart('regularAppStart'); +import '../../../index'; +Performance.markEnd('regularAppStart'); diff --git a/src/libs/E2E/tests/openSearchPageTest.e2e.js b/src/libs/E2E/tests/openSearchPageTest.e2e.js index 2f0f72f35bdd..3b2d91322cf0 100644 --- a/src/libs/E2E/tests/openSearchPageTest.e2e.js +++ b/src/libs/E2E/tests/openSearchPageTest.e2e.js @@ -7,24 +7,41 @@ import CONST from '../../../CONST'; const test = () => { // check for login (if already logged in the action will simply resolve) + console.debug('[E2E] Logging in for search'); + E2ELogin().then((neededLogin) => { if (neededLogin) { // we don't want to submit the first login to the results return E2EClient.submitTestDone(); } + console.debug('[E2E] Logged in, getting search metrics and submitting them…'); + Performance.subscribeToMeasurements((entry) => { + if (entry.name === CONST.TIMING.SIDEBAR_LOADED) { + console.debug(`[E2E] Sidebar loaded, navigating to search route…`); + Navigation.navigate(ROUTES.SEARCH); + return; + } + + console.debug(`[E2E] Entry: ${JSON.stringify(entry)}`); if (entry.name !== CONST.TIMING.SEARCH_RENDER) { return; } + console.debug(`[E2E] Submitting!`); E2EClient.submitTestResults({ name: 'Open Search Page TTI', duration: entry.duration, - }).then(E2EClient.submitTestDone); + }) + .then(() => { + console.debug('[E2E] Done with search, exiting…'); + E2EClient.submitTestDone(); + }) + .catch((err) => { + console.debug('[E2E] Error while submitting test results:', err); + }); }); - - Navigation.navigate(ROUTES.SEARCH); }); }; diff --git a/tests/e2e/ADDING_TESTS.md b/tests/e2e/ADDING_TESTS.md index 39cdb97ebed0..7b1caa977a63 100644 --- a/tests/e2e/ADDING_TESTS.md +++ b/tests/e2e/ADDING_TESTS.md @@ -1,4 +1,51 @@ -# Add E2E Tests +# Adding new E2E Tests + +## Running your new test in development mode + +Typically you'd run all the tests with `npm run test:e2e` on your machine, +this will run the tests with some local settings, however that is not +optimal when you add a new test for which you want to quickly test if it works, as it +still runs the release version of the app. + +I recommend doing the following. + +> [!NOTE] +> All of the steps can be executed at once by running XXX (todo) + +1. Rename `./index.js` to `./appIndex.js` +2. Create a new `./index.js` with the following content: +```js +requrire("./src/libs/E2E/reactNativeLaunchingTest.js"); +``` +3. In `./src/libs/E2E/reactNativeLaunchingTest.js` change the main app import to the new `./appIndex.js` file: +```diff +- import '../../../index'; ++ import '../../../appIndex'; +``` + +> [!WARNING] +> Make sure to not commit these changes to the repository! + +Now you can start the metro bundler in e2e mode with: + +``` +CAPTURE_METRICS=TRUE E2E_Testing=true npm start -- --reset-cache +``` + +Then we can execute our test with: + +``` +npm run test:e2e -- --development --skipInstallDeps --buildMode skip --includes "My new test name" +``` + +> - `--development` will run the tests with a local config, which will run the tests with fewer iterations +> - `--skipInstallDeps` will skip the `npm install` step, which you probably don't need +> - `--buildMode skip` will skip rebuilding the app, and just run the existing app +> - `--includes "MyTestName"` will only run the test with the name "MyTestName" + + + +## Creating a new test Tests are executed on device, inside the app code. @@ -97,6 +144,10 @@ Done! When you now start the test runner, your new test will be executed as well ## Quickly test your test To check your new test you can simply run `npm run test:e2e`, which uses the -`--development` flag. This will run the tests on the branch you are currently on -and will do fewer iterations. +`--development` flag. This will run the tests on the branch you are currently on, runs fewer iterations and most importantly, it tries to reuse the existing APK and just patch into the new app bundle, instead of rebuilding the release app from scratch. + +## Debugging your test + +You can use regular console statements to debug your test. The output will be visible +in logcat. I recommend opening the android studio logcat window and filter for `ReactNativeJS` to see the output you'd otherwise typically see in your metro bundler instance. diff --git a/tests/e2e/config.js b/tests/e2e/config.js index d322fb970b2d..d7844a29f3e4 100644 --- a/tests/e2e/config.js +++ b/tests/e2e/config.js @@ -21,7 +21,7 @@ const TEST_NAMES = { * ``` */ module.exports = { - APP_PACKAGE: 'com.expensify.chat', + APP_PACKAGE: 'com.expensify.chat.adhoc', APP_PATHS: { baseline: './app-e2eRelease-baseline.apk', diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js index cd0b04d7c3cf..0c38c3f1056f 100644 --- a/tests/e2e/config.local.js +++ b/tests/e2e/config.local.js @@ -1,8 +1,10 @@ module.exports = { + APP_PACKAGE: 'com.expensify.chat.dev', + WARM_UP_RUNS: 1, RUNS: 8, APP_PATHS: { - baseline: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk', - compare: './android/app/build/outputs/apk/e2eRelease/app-e2eRelease.apk', + baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', + compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', }, }; diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index db421ae64ef1..2a5aee78715f 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -73,9 +73,9 @@ if (isDevMode) { const restartApp = async () => { Logger.log('Killing app …'); - await killApp('android'); + await killApp('android', config.APP_PACKAGE); Logger.log('Launching app …'); - await launchApp('android'); + await launchApp('android', config.APP_PACKAGE); }; const runTestsOnBranch = async (baselineOrCompare, branch) => { @@ -89,6 +89,8 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { const appExists = fs.existsSync(appPath); if (!appExists) { Logger.warn(`Build mode "${buildMode}" is not possible, because the app does not exist. Falling back to build mode "full".`); + Logger.note(`App path: ${appPath}`); + buildMode = 'full'; } } @@ -125,7 +127,7 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { // Install app and reverse port let progressLog = Logger.progressInfo('Installing app and reversing port'); - await installApp('android', appPath); + await installApp('android', config.APP_PACKAGE, appPath); await reversePort(); progressLog.done(); diff --git a/tests/e2e/utils/installApp.js b/tests/e2e/utils/installApp.js index 136602375f85..ff961940826a 100644 --- a/tests/e2e/utils/installApp.js +++ b/tests/e2e/utils/installApp.js @@ -7,16 +7,17 @@ const Logger = require('./logger'); * It removes the app first if it already exists, so it's a clean installation. * * @param {String} platform + * @param {String} packageName * @param {String} path * @returns {Promise} */ -module.exports = function (platform = 'android', path) { +module.exports = function (platform = 'android', packageName = APP_PACKAGE, path) { if (platform !== 'android') { throw new Error(`installApp() missing implementation for platform: ${platform}`); } // Uninstall first, then install - return execAsync(`adb uninstall ${APP_PACKAGE}`) + return execAsync(`adb uninstall ${packageName}`) .catch((e) => { // Ignore errors Logger.warn('Failed to uninstall app:', e); diff --git a/tests/e2e/utils/killApp.js b/tests/e2e/utils/killApp.js index 9761ee7fc66e..bdef215bf752 100644 --- a/tests/e2e/utils/killApp.js +++ b/tests/e2e/utils/killApp.js @@ -1,11 +1,11 @@ const {APP_PACKAGE} = require('../config'); const execAsync = require('./execAsync'); -module.exports = function (platform = 'android') { +module.exports = function (platform = 'android', packageName = APP_PACKAGE) { if (platform !== 'android') { throw new Error(`killApp() missing implementation for platform: ${platform}`); } // Use adb to kill the app - return execAsync(`adb shell am force-stop ${APP_PACKAGE}`); + return execAsync(`adb shell am force-stop ${packageName}`); }; diff --git a/tests/e2e/utils/launchApp.js b/tests/e2e/utils/launchApp.js index dce17c7fbb3b..e0726d081086 100644 --- a/tests/e2e/utils/launchApp.js +++ b/tests/e2e/utils/launchApp.js @@ -1,11 +1,11 @@ const {APP_PACKAGE} = require('../config'); const execAsync = require('./execAsync'); -module.exports = function (platform = 'android') { +module.exports = function (platform = 'android', packageName = APP_PACKAGE) { if (platform !== 'android') { throw new Error(`launchApp() missing implementation for platform: ${platform}`); } // Use adb to start the app - return execAsync(`adb shell monkey -p ${APP_PACKAGE} -c android.intent.category.LAUNCHER 1`); + return execAsync(`adb shell monkey -p ${packageName} -c android.intent.category.LAUNCHER 1`); }; diff --git a/tests/e2e/utils/logger.js b/tests/e2e/utils/logger.js index aa198aec3004..1f2fff315bfc 100644 --- a/tests/e2e/utils/logger.js +++ b/tests/e2e/utils/logger.js @@ -61,19 +61,16 @@ const progressInfo = (textParam) => { }; const info = (...args) => { - console.debug('> ', ...args); - log(...args); + log('> ', ...args); }; const warn = (...args) => { const lines = [`\n${COLOR_YELLOW}⚠️`, ...args, `${COLOR_RESET}\n`]; - console.debug(...lines); log(...lines); }; const note = (...args) => { const lines = [`\n💡${COLOR_DIM}`, ...args, `${COLOR_RESET}\n`]; - console.debug(...lines); log(...lines); };