diff --git a/packages/kbn-test/src/functional_test_runner/cli.ts b/packages/kbn-test/src/functional_test_runner/cli.ts index 11b9450f2af6eb..5a3afc52e7eb94 100644 --- a/packages/kbn-test/src/functional_test_runner/cli.ts +++ b/packages/kbn-test/src/functional_test_runner/cli.ts @@ -48,6 +48,10 @@ export function runFtrCli() { kbnTestServer: { installDir: parseInstallDir(flags), }, + suiteFiles: { + include: toArray(flags['include-file'] as string | string[]), + exclude: toArray(flags['exclude-file'] as string | string[]), + }, suiteTags: { include: toArray(flags['include-tag'] as string | string[]), exclude: toArray(flags['exclude-tag'] as string | string[]), @@ -104,7 +108,16 @@ export function runFtrCli() { }, { flags: { - string: ['config', 'grep', 'exclude', 'include-tag', 'exclude-tag', 'kibana-install-dir'], + string: [ + 'config', + 'grep', + 'exclude', + 'include-file', + 'exclude-file', + 'include-tag', + 'exclude-tag', + 'kibana-install-dir', + ], boolean: ['bail', 'invert', 'test-stats', 'updateBaselines', 'throttle', 'headless'], default: { config: 'test/functional/config.js', @@ -116,6 +129,8 @@ export function runFtrCli() { --grep pattern used to select which tests to run --invert invert grep to exclude tests --exclude=file path to a test file that should not be loaded + --include-file=file a test file to be included, pass multiple times for multiple files + --exclude-file=file a test file to be excluded, pass multiple times for multiple files --include-tag=tag a tag to be included, pass multiple times for multiple tags --exclude-tag=tag a tag to be excluded, pass multiple times for multiple tags --test-stats print the number of tests (included and excluded) to STDERR diff --git a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts index 75623d6c088905..25889ac85c6dd9 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts +++ b/packages/kbn-test/src/functional_test_runner/lib/config/schema.ts @@ -68,6 +68,17 @@ export const schema = Joi.object() .items(Joi.string()) .default([]), + suiteFiles: Joi.object() + .keys({ + include: Joi.array() + .items(Joi.string()) + .default([]), + exclude: Joi.array() + .items(Joi.string()) + .default([]), + }) + .default(), + suiteTags: Joi.object() .keys({ include: Joi.array() diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js index 64fc51a04aac9b..c09ae5e775ea37 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ - +import { sep } from 'path'; +import { REPO_ROOT } from '@kbn/dev-utils'; import { createAssignmentProxy } from './assignment_proxy'; import { wrapFunction } from './wrap_function'; import { wrapRunnableArgs } from './wrap_runnable_args'; -export function decorateMochaUi(lifecycle, context) { +export function decorateMochaUi(lifecycle, context, config) { // incremented at the start of each suite, decremented after // so that in each non-suite call we can know if we are within // a suite, or that when a suite is defined it is within a suite @@ -65,6 +66,12 @@ export function decorateMochaUi(lifecycle, context) { this._tags = [].concat(this._tags || [], tags); }; + const filePath = this.file.replace(REPO_ROOT + sep, ''); + this.tags(filePath); + this.suiteTag = filePath; // The tag that uniquely targets this suite/file + + this.ftrConfig = config; + provider.call(this); after(async () => { diff --git a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js index 326877919d9855..980becf46ba992 100644 --- a/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js +++ b/packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js @@ -18,6 +18,9 @@ */ import Mocha from 'mocha'; +import { realpathSync } from 'fs'; +import { sep } from 'path'; +import { REPO_ROOT } from '@kbn/dev-utils'; import { loadTestFiles } from './load_test_files'; import { filterSuitesByTags } from './filter_suites_by_tags'; @@ -49,11 +52,23 @@ export async function setupMocha(lifecycle, log, config, providers) { log, lifecycle, providers, + config, paths: config.get('testFiles'), excludePaths: config.get('excludeTestFiles'), updateBaselines: config.get('updateBaselines'), }); + filterSuitesByTags({ + log, + mocha, + include: config + .get('suiteFiles.include') + .map(f => realpathSync(f).replace(REPO_ROOT + sep, '')), + exclude: config + .get('suiteFiles.exclude') + .map(f => realpathSync(f).replace(REPO_ROOT + sep, '')), + }); + filterSuitesByTags({ log, mocha, diff --git a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js index b34006a38a45d8..349816dc5d089f 100644 --- a/packages/kbn-test/src/functional_tests/cli/run_tests/args.js +++ b/packages/kbn-test/src/functional_tests/cli/run_tests/args.js @@ -46,6 +46,14 @@ const options = { updateBaselines: { desc: 'Replace baseline screenshots with whatever is generated from the test.', }, + 'include-file': { + arg: '', + desc: 'Files that must included to be run, can be included multiple times.', + }, + 'exclude-file': { + arg: '', + desc: 'Files that must NOT be included to be run, can be included multiple times.', + }, 'include-tag': { arg: '', desc: 'Tags that suites must include to be run, can be included multiple times.', @@ -115,6 +123,13 @@ export function processOptions(userOptions, defaultConfigPaths) { delete userOptions['kibana-install-dir']; } + userOptions.suiteFiles = { + include: [].concat(userOptions['include-file'] || []), + exclude: [].concat(userOptions['exclude-file'] || []), + }; + delete userOptions['include-file']; + delete userOptions['exclude-file']; + userOptions.suiteTags = { include: [].concat(userOptions['include-tag'] || []), exclude: [].concat(userOptions['exclude-tag'] || []), diff --git a/packages/kbn-test/src/functional_tests/lib/run_ftr.js b/packages/kbn-test/src/functional_tests/lib/run_ftr.js index 9b631e33f3b24c..14883ac977c431 100644 --- a/packages/kbn-test/src/functional_tests/lib/run_ftr.js +++ b/packages/kbn-test/src/functional_tests/lib/run_ftr.js @@ -22,7 +22,7 @@ import { CliError } from './run_cli'; async function createFtr({ configPath, - options: { installDir, log, bail, grep, updateBaselines, suiteTags }, + options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags }, }) { const config = await readConfigFile(log, configPath); @@ -37,6 +37,10 @@ async function createFtr({ installDir, }, updateBaselines, + suiteFiles: { + include: [...suiteFiles.include, ...config.get('suiteFiles.include')], + exclude: [...suiteFiles.exclude, ...config.get('suiteFiles.exclude')], + }, suiteTags: { include: [...suiteTags.include, ...config.get('suiteTags.include')], exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],