[Reporting/Test] move functional tests to apps

x-pack/scripts/functional_tests.js

@@ -14,7 +14,6 @@ const alwaysImportedTests = [
 ];
 const onlyNotInCoverageTests = [
   require.resolve('../test/reporting/configs/chromium_api.js'),
-  require.resolve('../test/reporting/configs/chromium_functional.js'),
   require.resolve('../test/reporting/configs/generate_api.js'),
   require.resolve('../test/api_integration/config_security_basic.js'),
   require.resolve('../test/api_integration/config.js'),

x-pack/test/functional/apps/dashboard/index.ts

@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/dashboard/reporting/README.md

@@ -0,0 +1,22 @@
+## The Dashboard Reporting Tests
+
+### Baseline snapshots
+
+The reporting tests create a few PNG reports and do a snapshot comparison against stored baselines.  The baseline images are stored in `./reports/baseline`.
+
+### Updating the baselines
+
+Every now and then visual changes will be made that will require the snapshots to be updated.  This is how you go about updating it.
+
+1. **Load the ES Archive containing the dashboard and data.**
+   This will load the test data into an Elasticsearch instance running via the functional test server:
+  ```
+  node scripts/es_archiver load reporting/ecommerce --config=x-pack/test/functional/config.js
+  node scripts/es_archiver load reporting/ecommerce_kibana --config=x-pack/test/functional/config.js
+  ```
+2. **Generate the reports of the E-commerce dashboard in the Kibana UI.**
+   Navigate to `http://localhost:5620`, find the archived dashboard, and generate all the types of reports for which there are stored baseline images.
+3. **Download the reports, and save them into the `reports/baseline` folder.**
+   Change the names of the PNG/PDF files to overwrite the stored baselines.
+
+The next time functional tests run, the generated reports will be compared to the latest image that you have saved :bowtie:

x-pack/test/functional/apps/dashboard/reporting/index.js

@@ -0,0 +1,126 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import path from 'path';
+import fs from 'fs';
+import { promisify } from 'util';
+import { checkIfPngsMatch } from './lib/compare_pngs';
+
+const writeFileAsync = promisify(fs.writeFile);
+const mkdirAsync = promisify(fs.mkdir);
+
+const REPORTS_FOLDER = path.resolve(__dirname, 'reports');
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const config = getService('config');
+  const PageObjects = getPageObjects(['reporting', 'common', 'dashboard']);
+
+  describe('Reporting', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PDF Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Print Layout', () => {
+      it('downloads a PDF file', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout because the comparePngs
+        // function is taking about 15 seconds per comparison in jenkins.
+        this.timeout(300000);
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.checkUsePrintLayout();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+
+    describe('Print PNG button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.clickNewDashboard();
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.dashboard.saveDashboard('My PNG Dash');
+        await PageObjects.reporting.openPngReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+    });
+
+    describe('Preserve Layout', () => {
+      it('matches baseline report', async function() {
+        const writeSessionReport = async (name, rawPdf, reportExt) => {
+          const sessionDirectory = path.resolve(REPORTS_FOLDER, 'session');
+          await mkdirAsync(sessionDirectory, { recursive: true });
+          const sessionReportPath = path.resolve(sessionDirectory, `${name}.${reportExt}`);
+          await writeFileAsync(sessionReportPath, rawPdf);
+          return sessionReportPath;
+        };
+        const getBaselineReportPath = (fileName, reportExt) => {
+          const baselineFolder = path.resolve(REPORTS_FOLDER, 'baseline');
+          const fullPath = path.resolve(baselineFolder, `${fileName}.${reportExt}`);
+          log.debug(`getBaselineReportPath (${fullPath})`);
+          return fullPath;
+        };
+
+        this.timeout(300000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPngReportingPanel();
+        await PageObjects.reporting.forceSharedItemsContainerSize({ width: 1405 });
+        await PageObjects.reporting.clickGenerateReportButton();
+        await PageObjects.reporting.removeForceSharedItemsContainerSize();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const reportData = await PageObjects.reporting.getRawPdfReportData(url);
+        const reportFileName = 'dashboard_preserve_layout';
+        const sessionReportPath = await writeSessionReport(reportFileName, reportData, 'png');
+        const percentSimilar = await checkIfPngsMatch(
+          sessionReportPath,
+          getBaselineReportPath(reportFileName, 'png'),
+          config.get('screenshots.directory'),
+          log
+        );
+
+        expect(percentSimilar).to.be.lessThan(0.1);
+      });
+    });
+  });
+}

x-pack/test/reporting/functional/lib/compare_pngs.js → x-pack/test/functional/apps/dashboard/reporting/lib/compare_pngs.js

@@ -7,7 +7,7 @@
 import path from 'path';
 import fs from 'fs';
 import { promisify } from 'bluebird';
-import { comparePngs } from '../../../../../test/functional/services/lib/compare_pngs';
+import { comparePngs } from '../../../../../../../test/functional/services/lib/compare_pngs';
 
 const mkdirAsync = promisify(fs.mkdir);
 

x-pack/test/functional/apps/discover/index.ts

@@ -11,5 +11,6 @@ export default function({ loadTestFile }: FtrProviderContext) {
 
     loadTestFile(require.resolve('./feature_controls'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/discover/reporting.js

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const log = getService('log');
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const PageObjects = getPageObjects(['reporting', 'common', 'discover']);
+  const filterBar = getService('filterBar');
+
+  describe('Discover', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+    });
+
+    describe('Generate CSV button', () => {
+      beforeEach(() => PageObjects.common.navigateToApp('discover'));
+
+      it('is not available if new', async () => {
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('becomes available/not available when a saved search is created, changed and saved again', async () => {
+        // create new search, csv export is not available
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+        // add filter, csv export is not available
+        await filterBar.addFilter('currency', 'is', 'EUR');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+        // save search again, csv export is available
+        await PageObjects.discover.saveSearch('my search - expectEnabledGenerateReportButton 2');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('generates a report with data', async () => {
+        await PageObjects.discover.clickNewSearchButton();
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.discover.saveSearch('my search - with data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+
+      it('generates a report with no data', async () => {
+        await PageObjects.reporting.setTimepickerInNoDataRange();
+        await PageObjects.discover.saveSearch('my search - no data - expectReportCanBeCreated');
+        await PageObjects.reporting.openCsvReportingPanel();
+        expect(await PageObjects.reporting.canReportBeCreated()).to.be(true);
+      });
+    });
+  });
+}

x-pack/test/functional/apps/visualize/index.ts

@@ -15,5 +15,6 @@ export default function visualize({ loadTestFile }: FtrProviderContext) {
     loadTestFile(require.resolve('./hybrid_visualization'));
     loadTestFile(require.resolve('./precalculated_histogram'));
     loadTestFile(require.resolve('./preserve_url'));
+    loadTestFile(require.resolve('./reporting'));
   });
 }

x-pack/test/functional/apps/visualize/reporting.js

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+
+export default function({ getService, getPageObjects }) {
+  const esArchiver = getService('esArchiver');
+  const browser = getService('browser');
+  const log = getService('log');
+  const PageObjects = getPageObjects([
+    'reporting',
+    'common',
+    'dashboard',
+    'visualize',
+    'visEditor',
+  ]);
+
+  describe('Visualize', () => {
+    before('initialize tests', async () => {
+      log.debug('ReportingPage:initTests');
+      await esArchiver.loadIfNeeded('reporting/ecommerce');
+      await esArchiver.loadIfNeeded('reporting/ecommerce_kibana');
+      await browser.setWindowSize(1600, 850);
+    });
+    after('clean up archives', async () => {
+      await esArchiver.unload('reporting/ecommerce');
+      await esArchiver.unload('reporting/ecommerce_kibana');
+    });
+
+    describe('Print PDF button', () => {
+      it('is not available if new', async () => {
+        await PageObjects.common.navigateToUrl('visualize', 'new');
+        await PageObjects.visualize.clickAreaChart();
+        await PageObjects.visualize.clickNewSearch('ecommerce');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be('true');
+      });
+
+      it('becomes available when saved', async () => {
+        await PageObjects.reporting.setTimepickerInDataRange();
+        await PageObjects.visEditor.clickBucket('X-axis');
+        await PageObjects.visEditor.selectAggregation('Date Histogram');
+        await PageObjects.visEditor.clickGo();
+        await PageObjects.visualize.saveVisualization('my viz');
+        await PageObjects.reporting.openPdfReportingPanel();
+        expect(await PageObjects.reporting.isGenerateReportButtonDisabled()).to.be(null);
+      });
+
+      it('downloaded PDF has OK status', async function() {
+        // Generating and then comparing reports can take longer than the default 60s timeout
+        this.timeout(180000);
+
+        await PageObjects.common.navigateToApp('dashboard');
+        await PageObjects.dashboard.loadSavedDashboard('Ecom Dashboard');
+        await PageObjects.reporting.openPdfReportingPanel();
+        await PageObjects.reporting.clickGenerateReportButton();
+
+        const url = await PageObjects.reporting.getReportURL(60000);
+        const res = await PageObjects.reporting.getResponse(url);
+
+        expect(res.statusCode).to.equal(200);
+        expect(res.headers['content-type']).to.equal('application/pdf');
+      });
+    });
+  });
+}

x-pack/test/functional/page_objects/reporting_page.js

@@ -17,7 +17,6 @@ import http from 'http';
 export function ReportingPageProvider({ getService, getPageObjects }) {
   const retry = getService('retry');
   const log = getService('log');
-  const config = getService('config');
   const testSubjects = getService('testSubjects');
   const browser = getService('browser');
   const PageObjects = getPageObjects(['common', 'security', 'share', 'timePicker']);
@@ -51,7 +50,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
 
     getResponse(url) {
       log.debug(`getResponse for ${url}`);
-      const auth = config.get('servers.elasticsearch.auth');
+      const auth = 'test_user:changeme'; // FIXME not sure why there is no config that can be read for this
       const headers = {
         Authorization: `Basic ${Buffer.from(auth).toString('base64')}`,
       };
@@ -71,6 +70,7 @@ export function ReportingPageProvider({ getService, getPageObjects }) {
             }
           )
           .on('error', e => {
+            log.error(e);
             reject(e);
           });
       });