From 4c460b8c429f4fa73f2ed9ca9c1725f0f9657d65 Mon Sep 17 00:00:00 2001 From: Tim Sullivan Date: Mon, 27 Apr 2020 14:24:55 -0700 Subject: [PATCH] [Reporting/Test] move functional tests to apps (#64368) * Squashed commit of the following: commit 5953089c03bea6b2d091f7723fea25bb1c210ee8 Author: Timothy Sullivan Date: Thu Apr 9 18:29:55 2020 -0700 move tests to apps commit 39adeaae6d3502d2c6da4e6111c2a396f2a7aedb Author: Timothy Sullivan Date: Thu Apr 9 17:49:20 2020 -0700 update archive with better dashboard commit 55b600748356c1adc5e75669b0c5588a812e401d Author: Timothy Sullivan Date: Thu Apr 9 17:16:53 2020 -0700 fix the refactoring bugs commit 11aff10cd8009aeb9bb78aa5ce0b37a72d47776e Author: Timothy Sullivan Date: Thu Apr 9 17:16:28 2020 -0700 remove unused fixtuers commit 05c33817c8eb67c461ac012cf2f71a9c01f1a91e Author: Timothy Sullivan Date: Thu Apr 9 16:37:36 2020 -0700 Start of refactoring commit b63c182b5f32b19dc3ca715efdbc5f18fcc02f67 Author: Timothy Sullivan Date: Thu Apr 9 16:32:50 2020 -0700 Todo comments commit 1e0105e673607b9b2581b2a47a24653fe545f97e Author: Timothy Sullivan Date: Thu Apr 9 14:31:58 2020 -0700 revert unrelated change commit 206fd14b7749b2c56ec4f36f0531214329942ff9 Merge: 0d4c2ad29a 834306458a Author: Timothy Sullivan Date: Thu Apr 9 14:28:45 2020 -0700 Merge branch 'master' into reporting/test-better commit 0d4c2ad29a3dc83d3ded404c15360c2cd4cdf260 Author: Timothy Sullivan Date: Wed Apr 8 10:41:19 2020 -0700 fix ts commit 890128c47d1fd561cd2c1393f7fa733e02e060ef Merge: d9ce4024ec 3598b8c44c Author: Timothy Sullivan Date: Wed Apr 8 10:31:09 2020 -0700 Merge branch 'master' into reporting/test-better commit d9ce4024ecde27974046dc5ae4ecf27a5580f308 Author: Timothy Sullivan Date: Tue Apr 7 08:31:58 2020 -0700 [Reporting] convert all server unit tests to TypeScript * fix imports and readmes * remove not-needed readme * remove extra info from readme * correct some comments * log the error that was caught * fix config path in readme * fix readme instructions to point to updated paths Co-authored-by: Elastic Machine --- x-pack/scripts/functional_tests.js | 1 - .../test/functional/apps/dashboard/index.ts | 1 + .../apps/dashboard/reporting/README.md | 22 ++ .../apps/dashboard/reporting/index.js | 126 +++++++++ .../dashboard/reporting}/lib/compare_pngs.js | 2 +- .../baseline/dashboard_preserve_layout.png | Bin x-pack/test/functional/apps/discover/index.ts | 1 + .../functional/apps/discover/reporting.js | 75 ++++++ .../test/functional/apps/visualize/index.ts | 1 + .../functional/apps/visualize/reporting.js | 69 +++++ .../functional/page_objects/reporting_page.js | 4 +- x-pack/test/reporting/README.md | 153 ----------- .../reporting/configs/chromium_functional.js | 39 --- x-pack/test/reporting/functional/index.js | 12 - x-pack/test/reporting/functional/lib/index.js | 7 - x-pack/test/reporting/functional/reporting.js | 251 ------------------ 16 files changed, 298 insertions(+), 466 deletions(-) create mode 100644 x-pack/test/functional/apps/dashboard/reporting/README.md create mode 100644 x-pack/test/functional/apps/dashboard/reporting/index.js rename x-pack/test/{reporting/functional => functional/apps/dashboard/reporting}/lib/compare_pngs.js (95%) rename x-pack/test/{reporting/functional => functional/apps/dashboard/reporting}/reports/baseline/dashboard_preserve_layout.png (100%) create mode 100644 x-pack/test/functional/apps/discover/reporting.js create mode 100644 x-pack/test/functional/apps/visualize/reporting.js delete mode 100644 x-pack/test/reporting/README.md delete mode 100644 x-pack/test/reporting/configs/chromium_functional.js delete mode 100644 x-pack/test/reporting/functional/index.js delete mode 100644 x-pack/test/reporting/functional/lib/index.js delete mode 100644 x-pack/test/reporting/functional/reporting.js diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index 974f3eb6db60ff..efe1f859059706 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -14,7 +14,6 @@ const alwaysImportedTests = [ ]; const onlyNotInCoverageTests = [ require.resolve('../test/reporting/configs/chromium_api.js'), - require.resolve('../test/reporting/configs/chromium_functional.js'), require.resolve('../test/reporting/configs/generate_api.js'), require.resolve('../test/api_integration/config_security_basic.js'), require.resolve('../test/api_integration/config.js'), diff --git a/x-pack/test/functional/apps/dashboard/index.ts b/x-pack/test/functional/apps/dashboard/index.ts index 794bd2f1f0db36..23825836caad32 100644 --- a/x-pack/test/functional/apps/dashboard/index.ts +++ b/x-pack/test/functional/apps/dashboard/index.ts @@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./preserve_url')); + loadTestFile(require.resolve('./reporting')); }); } diff --git a/x-pack/test/functional/apps/dashboard/reporting/README.md b/x-pack/test/functional/apps/dashboard/reporting/README.md new file mode 100644 index 00000000000000..3a2b8f5cc783f9 --- /dev/null +++ b/x-pack/test/functional/apps/dashboard/reporting/README.md @@ -0,0 +1,22 @@ +## The Dashboard Reporting Tests + +### Baseline snapshots + +The reporting tests create a few PNG reports and do a snapshot comparison against stored baselines. The baseline images are stored in `./reports/baseline`. + +### Updating the baselines + +Every now and then visual changes will be made that will require the snapshots to be updated. This is how you go about updating it. + +1. **Load the ES Archive containing the dashboard and data.** + This will load the test data into an Elasticsearch instance running via the functional test server: + ``` + node scripts/es_archiver load reporting/ecommerce --config=x-pack/test/functional/config.js + node scripts/es_archiver load reporting/ecommerce_kibana --config=x-pack/test/functional/config.js + ``` +2. **Generate the reports of the E-commerce dashboard in the Kibana UI.** + Navigate to `http://localhost:5620`, find the archived dashboard, and generate all the types of reports for which there are stored baseline images. +3. **Download the reports, and save them into the `reports/baseline` folder.** + Change the names of the PNG/PDF files to overwrite the stored baselines. + +The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie: \ No newline at end of file diff --git a/x-pack/test/functional/apps/dashboard/reporting/index.js b/x-pack/test/functional/apps/dashboard/reporting/index.js new file mode 100644 index 00000000000000..99be084d80d74c --- /dev/null +++ b/x-pack/test/functional/apps/dashboard/reporting/index.js @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import path from 'path'; +import fs from 'fs'; +import { promisify } from 'util'; +import { checkIfPngsMatch } from './lib/compare_pngs'; + +const writeFileAsync = promisify(fs.writeFile); +const mkdirAsync = promisify(fs.mkdir); + +const REPORTS_FOLDER = path.resolve(__dirname, 'reports'); + +export default function({ getService, getPageObjects }) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + const log = getService('log'); + const config = getService('config'); + const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']); + + describe('Reporting', () => { + before('initialize tests', async () => { + log.debug('ReportingPage:initTests'); + await esArchiver.loadIfNeeded('reporting/ecommerce'); + await esArchiver.loadIfNeeded('reporting/ecommerce_kibana'); + await browser.setWindowSize(1600, 850); + }); + after('clean up archives', async () => { + await esArchiver.unload('reporting/ecommerce'); + await esArchiver.unload('reporting/ecommerce_kibana'); + }); + + describe('Print PDF button', () => { + it('is not available if new', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + await PageObjects.reporting.openPdfReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + }); + + it('becomes available when saved', async () => { + await PageObjects.dashboard.saveDashboard('My PDF Dashboard'); + await PageObjects.reporting.openPdfReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + }); + }); + + describe('Print Layout', () => { + it('downloads a PDF file', async function() { + // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs + // function is taking about 15 seconds per comparison in jenkins. + this.timeout(300000); + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); + await PageObjects.reporting.openPdfReportingPanel(); + await PageObjects.reporting.checkUsePrintLayout(); + await PageObjects.reporting.clickGenerateReportButton(); + + const url = await PageObjects.reporting.getReportURL(60000); + const res = await PageObjects.reporting.getResponse(url); + + expect(res.statusCode).to.equal(200); + expect(res.headers['content-type']).to.equal('application/pdf'); + }); + }); + + describe('Print PNG button', () => { + it('is not available if new', async () => { + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.clickNewDashboard(); + await PageObjects.reporting.openPngReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + }); + + it('becomes available when saved', async () => { + await PageObjects.dashboard.saveDashboard('My PNG Dash'); + await PageObjects.reporting.openPngReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + }); + }); + + describe('Preserve Layout', () => { + it('matches baseline report', async function() { + const writeSessionReport = async (name, rawPdf, reportExt) => { + const sessionDirectory = path.resolve(REPORTS_FOLDER, 'session'); + await mkdirAsync(sessionDirectory, { recursive: true }); + const sessionReportPath = path.resolve(sessionDirectory, `${name}.${reportExt}`); + await writeFileAsync(sessionReportPath, rawPdf); + return sessionReportPath; + }; + const getBaselineReportPath = (fileName, reportExt) => { + const baselineFolder = path.resolve(REPORTS_FOLDER, 'baseline'); + const fullPath = path.resolve(baselineFolder, `${fileName}.${reportExt}`); + log.debug(`getBaselineReportPath (${fullPath})`); + return fullPath; + }; + + this.timeout(300000); + + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); + await PageObjects.reporting.openPngReportingPanel(); + await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 }); + await PageObjects.reporting.clickGenerateReportButton(); + await PageObjects.reporting.removeForceSharedItemsContainerSize(); + + const url = await PageObjects.reporting.getReportURL(60000); + const reportData = await PageObjects.reporting.getRawPdfReportData(url); + const reportFileName = 'dashboard_preserve_layout'; + const sessionReportPath = await writeSessionReport(reportFileName, reportData, 'png'); + const percentSimilar = await checkIfPngsMatch( + sessionReportPath, + getBaselineReportPath(reportFileName, 'png'), + config.get('screenshots.directory'), + log + ); + + expect(percentSimilar).to.be.lessThan(0.1); + }); + }); + }); +} diff --git a/x-pack/test/reporting/functional/lib/compare_pngs.js b/x-pack/test/functional/apps/dashboard/reporting/lib/compare_pngs.js similarity index 95% rename from x-pack/test/reporting/functional/lib/compare_pngs.js rename to x-pack/test/functional/apps/dashboard/reporting/lib/compare_pngs.js index 16b1008c645a05..13c97a7fce7858 100644 --- a/x-pack/test/reporting/functional/lib/compare_pngs.js +++ b/x-pack/test/functional/apps/dashboard/reporting/lib/compare_pngs.js @@ -7,7 +7,7 @@ import path from 'path'; import fs from 'fs'; import { promisify } from 'bluebird'; -import { comparePngs } from '../../../../../test/functional/services/lib/compare_pngs'; +import { comparePngs } from '../../../../../../../test/functional/services/lib/compare_pngs'; const mkdirAsync = promisify(fs.mkdir); diff --git a/x-pack/test/reporting/functional/reports/baseline/dashboard_preserve_layout.png b/x-pack/test/functional/apps/dashboard/reporting/reports/baseline/dashboard_preserve_layout.png similarity index 100% rename from x-pack/test/reporting/functional/reports/baseline/dashboard_preserve_layout.png rename to x-pack/test/functional/apps/dashboard/reporting/reports/baseline/dashboard_preserve_layout.png diff --git a/x-pack/test/functional/apps/discover/index.ts b/x-pack/test/functional/apps/discover/index.ts index 2e5bdd736337d7..7688315a005168 100644 --- a/x-pack/test/functional/apps/discover/index.ts +++ b/x-pack/test/functional/apps/discover/index.ts @@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./feature_controls')); loadTestFile(require.resolve('./preserve_url')); + loadTestFile(require.resolve('./reporting')); }); } diff --git a/x-pack/test/functional/apps/discover/reporting.js b/x-pack/test/functional/apps/discover/reporting.js new file mode 100644 index 00000000000000..4aa005fc2db553 --- /dev/null +++ b/x-pack/test/functional/apps/discover/reporting.js @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function({ getService, getPageObjects }) { + const log = getService('log'); + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + const PageObjects = getPageObjects(['reporting', 'common', 'discover']); + const filterBar = getService('filterBar'); + + describe('Discover', () => { + before('initialize tests', async () => { + log.debug('ReportingPage:initTests'); + await esArchiver.loadIfNeeded('reporting/ecommerce'); + await browser.setWindowSize(1600, 850); + }); + after('clean up archives', async () => { + await esArchiver.unload('reporting/ecommerce'); + }); + + describe('Generate CSV button', () => { + beforeEach(() => PageObjects.common.navigateToApp('discover')); + + it('is not available if new', async () => { + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + }); + + it('becomes available when saved', async () => { + await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + }); + + it('becomes available/not available when a saved search is created, changed and saved again', async () => { + // create new search, csv export is not available + await PageObjects.discover.clickNewSearchButton(); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + // save search, csv export is available + await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + // add filter, csv export is not available + await filterBar.addFilter('currency', 'is', 'EUR'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + // save search again, csv export is available + await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + }); + + it('generates a report with data', async () => { + await PageObjects.discover.clickNewSearchButton(); + await PageObjects.reporting.setTimepickerInDataRange(); + await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.canReportBeCreated()).to.be(true); + }); + + it('generates a report with no data', async () => { + await PageObjects.reporting.setTimepickerInNoDataRange(); + await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated'); + await PageObjects.reporting.openCsvReportingPanel(); + expect(await PageObjects.reporting.canReportBeCreated()).to.be(true); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/visualize/index.ts b/x-pack/test/functional/apps/visualize/index.ts index f30367ba3dd0b3..1c23b8cde86064 100644 --- a/x-pack/test/functional/apps/visualize/index.ts +++ b/x-pack/test/functional/apps/visualize/index.ts @@ -15,5 +15,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./hybrid_visualization')); loadTestFile(require.resolve('./precalculated_histogram')); loadTestFile(require.resolve('./preserve_url')); + loadTestFile(require.resolve('./reporting')); }); } diff --git a/x-pack/test/functional/apps/visualize/reporting.js b/x-pack/test/functional/apps/visualize/reporting.js new file mode 100644 index 00000000000000..bc252e1ad4134b --- /dev/null +++ b/x-pack/test/functional/apps/visualize/reporting.js @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +export default function({ getService, getPageObjects }) { + const esArchiver = getService('esArchiver'); + const browser = getService('browser'); + const log = getService('log'); + const PageObjects = getPageObjects([ + 'reporting', + 'common', + 'dashboard', + 'visualize', + 'visEditor', + ]); + + describe('Visualize', () => { + before('initialize tests', async () => { + log.debug('ReportingPage:initTests'); + await esArchiver.loadIfNeeded('reporting/ecommerce'); + await esArchiver.loadIfNeeded('reporting/ecommerce_kibana'); + await browser.setWindowSize(1600, 850); + }); + after('clean up archives', async () => { + await esArchiver.unload('reporting/ecommerce'); + await esArchiver.unload('reporting/ecommerce_kibana'); + }); + + describe('Print PDF button', () => { + it('is not available if new', async () => { + await PageObjects.common.navigateToUrl('visualize', 'new'); + await PageObjects.visualize.clickAreaChart(); + await PageObjects.visualize.clickNewSearch('ecommerce'); + await PageObjects.reporting.openPdfReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); + }); + + it('becomes available when saved', async () => { + await PageObjects.reporting.setTimepickerInDataRange(); + await PageObjects.visEditor.clickBucket('X-axis'); + await PageObjects.visEditor.selectAggregation('Date Histogram'); + await PageObjects.visEditor.clickGo(); + await PageObjects.visualize.saveVisualization('my viz'); + await PageObjects.reporting.openPdfReportingPanel(); + expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); + }); + + it('downloaded PDF has OK status', async function() { + // Generating and then comparing reports can take longer than the default 60s timeout + this.timeout(180000); + + await PageObjects.common.navigateToApp('dashboard'); + await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); + await PageObjects.reporting.openPdfReportingPanel(); + await PageObjects.reporting.clickGenerateReportButton(); + + const url = await PageObjects.reporting.getReportURL(60000); + const res = await PageObjects.reporting.getResponse(url); + + expect(res.statusCode).to.equal(200); + expect(res.headers['content-type']).to.equal('application/pdf'); + }); + }); + }); +} diff --git a/x-pack/test/functional/page_objects/reporting_page.js b/x-pack/test/functional/page_objects/reporting_page.js index 35a76c74d4811c..b24ba8cf95d1ce 100644 --- a/x-pack/test/functional/page_objects/reporting_page.js +++ b/x-pack/test/functional/page_objects/reporting_page.js @@ -17,7 +17,6 @@ import http from 'http'; export function ReportingPageProvider({ getService, getPageObjects }) { const retry = getService('retry'); const log = getService('log'); - const config = getService('config'); const testSubjects = getService('testSubjects'); const browser = getService('browser'); const PageObjects = getPageObjects(['common', 'security', 'share', 'timePicker']); @@ -51,7 +50,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) { getResponse(url) { log.debug(`getResponse for ${url}`); - const auth = config.get('servers.elasticsearch.auth'); + const auth = 'test_user:changeme'; // FIXME not sure why there is no config that can be read for this const headers = { Authorization: `Basic ${Buffer.from(auth).toString('base64')}`, }; @@ -71,6 +70,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) { } ) .on('error', e => { + log.error(e); reject(e); }); }); diff --git a/x-pack/test/reporting/README.md b/x-pack/test/reporting/README.md deleted file mode 100644 index fc5147ad8c4544..00000000000000 --- a/x-pack/test/reporting/README.md +++ /dev/null @@ -1,153 +0,0 @@ -## The Reporting Tests - -### Overview - -Reporting tests have their own top level test folder because: - - Current API tests run with `optimize.enabled=false` flag for performance reasons, but reporting actually requires UI assets. - - Reporting tests take a lot longer than other test types. This separation allows developers to run them in isolation, or to run other functional or API tests without them. - - ### Running the tests - - There is more information on running x-pack tests here: https://github.com/elastic/kibana/blob/master/x-pack/README.md#running-functional-tests. Similar to running the API tests, you need to specify a reporting configuration file. Reporting currently has two configuration files you can point to: - - test/reporting/configs/chromium_api.js - - test/reporting/configs/chromium_functional.js - - The `api` versions hit the reporting api and ensure report generation completes successfully, but does not verify the output of the reports. This is done in the `functional` test versions, which does a snapshot comparison of the generated URL against a baseline to determine success. - - To run the tests in a single command. : -1. cd into x-pack directory. -2. run: - ``` -node scripts/functional_tests --config test/reporting/configs/[config_file_name_here].js - ``` - - You can also run the test server seperately from the runner. This is beneficial when debugging as Kibana and Elasticsearch will remain up and running throughout multiple test runs. To do this: - -1. cd into x-pack directory. -2. In one terminal window, run: - ``` -node scripts/functional_tests_server.js --config test/reporting/configs/[test_config_name_here].js - ``` -3. In another terminal window, cd into x-pack dir and run: - ``` -node ../scripts/functional_test_runner.js --config test/reporting/configs/[test_config_name_here].js - ``` - -```sh -//OSX -brew install imagemagick ghostscript poppler - -//Ubutnu -sudo apt-get install imagemagick ghostscript poppler-utils -``` - -For windows: - -ImageMagick-6.9.9-37-Q16-HDRI-x64-dll.exe -from -https://sourceforge.net/projects/imagemagick/postdownload -Install with all default options - -gs925w64.exe -from -https://github.com/ArtifexSoftware/ghostpdl-downloads/releases/download/gs925/gs925w64.exe -Install with all default options - - -**Note:** Configurations from `kibana.dev.yml` are picked up when running the tests. Ensure that `kibana.dev.yml` does not contain any `xpack.reporting` configurations. - -### Reporting baseline snapshots - -The functional version of the reporting tests create a few pdf reports and do a snapshot comparison against a couple baselines. The baseline images are stored in `./functional/reports/baseline`. - -#### Updating the baselines - -Every now and then visual changes will be made that will require the snapshots to be updated. This is how you go about updating it. I will discuss generating snapshots from chromium since that is the way of the future. - -1. Run the test server for chromium. - ``` -node scripts/functional_tests_server.js --config test/reporting/configs/chromium_functional.js - ``` - 2. Run the test runner - ``` - node ../scripts/functional_test_runner.js --config test/reporting/configs/chromium_functional.js - ``` - 3. This will create new report snapshots in `./functional/reports/session/`. - 4. After manually verifying the new reports in the `session` folder, copy them into [./functional/reports/baseline/](https://github.com/elastic/kibana/blob/master/x-pack/test/reporting/functional/reports/baseline) - 5. Create a new PR with the new snapshots in the baseline folder. - -**Note:** Dashboard has some snapshot testing too, in `_dashboard_snapshots.js`. This test watches for a command line flag `--updateBaselines` which automates updating the baselines. Probably worthwhile to do some similar here in the long run. - -``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 load ../../../../test/functional/fixtures/es_archiver/dashboard/current/kibana -``` -^^ That loads the .kibana index. - -``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 load ../../../../test/functional/fixtures/es_archiver/dashboard/current/data -``` -^^ That loads the data indices. - -**Note:** Depending on your kibana.yml configuration, you may need to adjust the username, pw, and port in the urls above. - -5. Navigate to Kibana in the browser (`http://localhost:5601`) -6. Log in, pick any index to be the default to get page the management screen (doesn’t matter) -7. Generate some reporting URLs - - Use a mixture of Visualize, Discover (CSV), Dashboard - - Can view the current test coverage by checkout out [api/generation_urls.js](https://github.com/elastic/kibana/blob/master/x-pack/test/reporting/api/generation_urls.js). You can use different ones for better test coverage (e.g. different dashboards, different visualizations). - - Don’t generate urls from huge dashboards since this is time consuming. - - Use dashboards that have time saved with them if you wish to have data included. -8. Save these reporting urls. -9. Navigate back to the main branch via `git checkout master`. Then create, or work off your branch as usual to add the extra test coverage. -10. Copy the urls into `api/generation_urls.js`, stripping out the domain, port and base path (if they have it). -11. Write your new tests in [api/bwc_generation_urls.js](https://github.com/elastic/kibana/blob/master/x-pack/test/reporting/api/bwc_generation_urls.js) -12. Run tests via the mechanism above. - -**Note:** We may at some point wish to push most of these tests into an integration suite, as testing BWC of urls generated in every single minor, especially if there were not notable changes, may be overkill, especially given the time they add to the ci. - -### Expanding test coverage by including more data - -As Kibana development progresses, our existing data indices often fail to cover new situations, such as: - - Reports with new visualization types - - Canvas reports - - Reports with visualizations referencing new index types (e.g. visualizations referencing rolled up indices) - - etc - - Every now and then we should expand our test coverage to include new features. This is how you go about doing that in the context of reporting: - - 1. Checkout the `[version].x` branch, where `version` is the currently working minor version (e.g. 6.x, not `master`). This is because we don't want to run tests from data generated from a master version. The opposite works fine however - tests on master can run against data generated from the last minor. At least generally, though major version upgrades may require updating archived data (or be run through the appropriate migration scripts). - - 2. Load the current archives via: - ``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 load ../../../../test/functional/fixtures/es_archiver/dashboard/current/kibana -``` -^^ That loads the .kibana index. - -``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 load ../../../../test/functional/fixtures/es_archiver/dashboard/current/data -``` -^^ That loads the data indices. - -**Note:** Your es-url parameter might be different, but those are the default ports if running via `yarn start` and `yarn es snapshot --license trial`. - -3. Now generate the new data, create new index patterns, create new visualizations, and create new dashboards using those visualizations. All the fun stuff that you may want to use in your tests. - -**Note:** This data is used in open source dashboard testing. All visualizations and saved searches that have `Rendering Test` in their name are dynamically added to a new dashboard and their rendering is confirmed in https://github.com/elastic/kibana/tree/master/test/functional/apps/dashboard/_embedddable_rendering.js. You may need to adjust the expectations if you add new tests (which will be a good thing anyway, help extend the basic rendering tests - this way issues are caught before it gets to reporting tests!). Similarly all visualizations and saved searches that have `Filter Bytes Test` in their name are tested in https://github.com/elastic/kibana/tree/master/test/functional/apps/dashboard/_dashboard_filtering.js - -**Note:** The current reporting tests add visualizations from what is in `PageObjects.dashboard.getTestVisualizationNames`. We should probably instead use a saved dashboard we generate this report from. Then you can add any new visualizations, re-save the dashboard, and re-generate the snapshot above. - -4. After adding more visualizations to a test dashboard, update tests if necessary, update snapshots, then **save the new archives**! - ``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 save ../../../../test/functional/fixtures/es_archiver/dashboard/current/kibana -``` -^^ That saves the .kibana index. - -``` -node ../scripts/es_archiver.js --es-url http://elastic:changeme@localhost:9200 save ../../../../test/functional/fixtures/es_archiver/dashboard/current/data log* animal* dog* [ANY OTHER NEW INDEX NAME YOU ADDED]* -``` -^^ That saves the data indices. The last parameter is a list of indices you want archived. You don't want to include the `.kibana` one in there (this way you can use a custom `.kibana`, but can reuse the data archive, for tests, and `.kibana` archive is small, but the data archives are larger and should be reused). - -5. Create your PR with test updates, and the larger archive. - - - diff --git a/x-pack/test/reporting/configs/chromium_functional.js b/x-pack/test/reporting/configs/chromium_functional.js deleted file mode 100644 index 753d2b2a20ab95..00000000000000 --- a/x-pack/test/reporting/configs/chromium_functional.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export default async function({ readConfigFile }) { - // TODO move reporting tests to x-pack/test/functional/apps//reporting - const functionalConfig = await readConfigFile(require.resolve('../../functional/config.js')); - - return { - services: functionalConfig.get('services'), - pageObjects: functionalConfig.get('pageObjects'), - servers: functionalConfig.get('servers'), - apps: functionalConfig.get('apps'), - screenshots: functionalConfig.get('screenshots'), - junit: { - reportName: 'X-Pack Chromium Functional Reporting Tests', - }, - testFiles: [require.resolve('../functional')], - kbnTestServer: { - ...functionalConfig.get('kbnTestServer'), - serverArgs: [ - ...functionalConfig.get('kbnTestServer.serverArgs'), - '--logging.events.log', - '["info","warning","error","fatal","optimize","reporting"]', - '--xpack.endpoint.enabled=true', - '--xpack.reporting.csv.enablePanelActionDownload=true', - '--xpack.reporting.csv.checkForFormulas=false', - '--xpack.reporting.csv.maxSizeBytes=25000000', - '--xpack.security.session.idleTimeout=3600000', - '--xpack.spaces.enabled=false', - ], - }, - esArchiver: functionalConfig.get('esArchiver'), - esTestCluster: functionalConfig.get('esTestCluster'), - security: { disableTestUser: true }, - }; -} diff --git a/x-pack/test/reporting/functional/index.js b/x-pack/test/reporting/functional/index.js deleted file mode 100644 index be3ac09bdec10b..00000000000000 --- a/x-pack/test/reporting/functional/index.js +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export default function({ loadTestFile }) { - describe('reporting app', function() { - this.tags('ciGroup6'); - loadTestFile(require.resolve('./reporting')); - }); -} diff --git a/x-pack/test/reporting/functional/lib/index.js b/x-pack/test/reporting/functional/lib/index.js deleted file mode 100644 index e7a08753b591f5..00000000000000 --- a/x-pack/test/reporting/functional/lib/index.js +++ /dev/null @@ -1,7 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export { checkIfPngsMatch } from './compare_pngs'; diff --git a/x-pack/test/reporting/functional/reporting.js b/x-pack/test/reporting/functional/reporting.js deleted file mode 100644 index c1a2ae634662ca..00000000000000 --- a/x-pack/test/reporting/functional/reporting.js +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import expect from '@kbn/expect'; -import path from 'path'; -import fs from 'fs'; -import { promisify } from 'util'; -import { checkIfPngsMatch } from './lib'; - -const writeFileAsync = promisify(fs.writeFile); -const mkdirAsync = promisify(fs.mkdir); - -const REPORTS_FOLDER = path.resolve(__dirname, 'reports'); - -/* - * TODO Remove this file and spread the tests to various apps - */ - -export default function({ getService, getPageObjects }) { - const esArchiver = getService('esArchiver'); - const browser = getService('browser'); - const log = getService('log'); - const config = getService('config'); - const filterBar = getService('filterBar'); - const PageObjects = getPageObjects([ - 'reporting', - 'common', - 'dashboard', - 'header', - 'discover', - 'visualize', - 'visEditor', - ]); - - describe('Reporting', () => { - describe('Dashboard', () => { - before('initialize tests', async () => { - log.debug('ReportingPage:initTests'); - await esArchiver.loadIfNeeded('reporting/ecommerce'); - await esArchiver.loadIfNeeded('reporting/ecommerce_kibana'); - await browser.setWindowSize(1600, 850); - }); - after('clean up archives', async () => { - await esArchiver.unload('reporting/ecommerce'); - await esArchiver.unload('reporting/ecommerce_kibana'); - }); - - describe('Print PDF button', () => { - it('is not available if new', async () => { - await PageObjects.common.navigateToApp('dashboard'); - await PageObjects.dashboard.clickNewDashboard(); - await PageObjects.reporting.openPdfReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - }); - - it('becomes available when saved', async () => { - await PageObjects.dashboard.saveDashboard('My PDF Dashboard'); - await PageObjects.reporting.openPdfReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - }); - }); - - describe('Print Layout', () => { - it('downloads a PDF file', async function() { - // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs - // function is taking about 15 seconds per comparison in jenkins. - this.timeout(300000); - await PageObjects.common.navigateToApp('dashboard'); - await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); - await PageObjects.reporting.openPdfReportingPanel(); - await PageObjects.reporting.checkUsePrintLayout(); - await PageObjects.reporting.clickGenerateReportButton(); - - const url = await PageObjects.reporting.getReportURL(60000); - const res = await PageObjects.reporting.getResponse(url); - - expect(res.statusCode).to.equal(200); - expect(res.headers['content-type']).to.equal('application/pdf'); - }); - }); - - describe('Print PNG button', () => { - it('is not available if new', async () => { - await PageObjects.common.navigateToApp('dashboard'); - await PageObjects.dashboard.clickNewDashboard(); - await PageObjects.reporting.openPngReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - }); - - it('becomes available when saved', async () => { - await PageObjects.dashboard.saveDashboard('My PNG Dash'); - await PageObjects.reporting.openPngReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - }); - }); - - describe('Preserve Layout', () => { - it('matches baseline report', async function() { - const writeSessionReport = async (name, rawPdf, reportExt) => { - const sessionDirectory = path.resolve(REPORTS_FOLDER, 'session'); - await mkdirAsync(sessionDirectory, { recursive: true }); - const sessionReportPath = path.resolve(sessionDirectory, `${name}.${reportExt}`); - await writeFileAsync(sessionReportPath, rawPdf); - return sessionReportPath; - }; - const getBaselineReportPath = (fileName, reportExt) => { - const baselineFolder = path.resolve(REPORTS_FOLDER, 'baseline'); - const fullPath = path.resolve(baselineFolder, `${fileName}.${reportExt}`); - log.debug(`getBaselineReportPath (${fullPath})`); - return fullPath; - }; - - this.timeout(300000); - - await PageObjects.common.navigateToApp('dashboard'); - await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); - await PageObjects.reporting.openPngReportingPanel(); - await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 }); - await PageObjects.reporting.clickGenerateReportButton(); - await PageObjects.reporting.removeForceSharedItemsContainerSize(); - - const url = await PageObjects.reporting.getReportURL(60000); - const reportData = await PageObjects.reporting.getRawPdfReportData(url); - const reportFileName = 'dashboard_preserve_layout'; - const sessionReportPath = await writeSessionReport(reportFileName, reportData, 'png'); - const percentSimilar = await checkIfPngsMatch( - sessionReportPath, - getBaselineReportPath(reportFileName, 'png'), - config.get('screenshots.directory'), - log - ); - - expect(percentSimilar).to.be.lessThan(0.1); - }); - }); - }); - - describe('Discover', () => { - before('initialize tests', async () => { - log.debug('ReportingPage:initTests'); - await esArchiver.loadIfNeeded('reporting/ecommerce'); - await browser.setWindowSize(1600, 850); - }); - after('clean up archives', async () => { - await esArchiver.unload('reporting/ecommerce'); - }); - - describe('Generate CSV button', () => { - beforeEach(() => PageObjects.common.navigateToApp('discover')); - - it('is not available if new', async () => { - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - }); - - it('becomes available when saved', async () => { - await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - }); - - it('becomes available/not available when a saved search is created, changed and saved again', async () => { - // create new search, csv export is not available - await PageObjects.discover.clickNewSearchButton(); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - // save search, csv export is available - await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - // add filter, csv export is not available - await filterBar.addFilter('currency', 'is', 'EUR'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - // save search again, csv export is available - await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - }); - - it('generates a report with data', async () => { - await PageObjects.discover.clickNewSearchButton(); - await PageObjects.reporting.setTimepickerInDataRange(); - await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.canReportBeCreated()).to.be(true); - }); - - it('generates a report with no data', async () => { - await PageObjects.reporting.setTimepickerInNoDataRange(); - await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated'); - await PageObjects.reporting.openCsvReportingPanel(); - expect(await PageObjects.reporting.canReportBeCreated()).to.be(true); - }); - }); - }); - - describe('Visualize', () => { - before('initialize tests', async () => { - log.debug('ReportingPage:initTests'); - await esArchiver.loadIfNeeded('reporting/ecommerce'); - await esArchiver.loadIfNeeded('reporting/ecommerce_kibana'); - await browser.setWindowSize(1600, 850); - }); - after('clean up archives', async () => { - await esArchiver.unload('reporting/ecommerce'); - await esArchiver.unload('reporting/ecommerce_kibana'); - }); - - describe('Print PDF button', () => { - it('is not available if new', async () => { - await PageObjects.common.navigateToUrl('visualize', 'new'); - await PageObjects.visualize.clickAreaChart(); - await PageObjects.visualize.clickNewSearch('ecommerce'); - await PageObjects.reporting.openPdfReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true'); - }); - - it('becomes available when saved', async () => { - await PageObjects.reporting.setTimepickerInDataRange(); - await PageObjects.visEditor.clickBucket('X-axis'); - await PageObjects.visEditor.selectAggregation('Date Histogram'); - await PageObjects.visEditor.clickGo(); - await PageObjects.visualize.saveVisualization('my viz'); - await PageObjects.reporting.openPdfReportingPanel(); - expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null); - }); - - it('downloaded PDF has OK status', async function() { - // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs - // function is taking about 15 seconds per comparison in jenkins. - this.timeout(180000); - - await PageObjects.common.navigateToApp('dashboard'); - await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard'); - await PageObjects.reporting.openPdfReportingPanel(); - await PageObjects.reporting.clickGenerateReportButton(); - - const url = await PageObjects.reporting.getReportURL(60000); - const res = await PageObjects.reporting.getResponse(url); - - expect(res.statusCode).to.equal(200); - expect(res.headers['content-type']).to.equal('application/pdf'); - }); - }); - }); - }); -}