tests(UI): add Argos CI

* tests(visual): add Argos CI
* tests(visual): fix feedbacks

.gitignore

@@ -5,3 +5,6 @@ npm-debug.log
 .DS_Store
 docs/
 !scripts/docs/
+
+# Ignore Argos CI screenshots.
+functional-tests/screenshots/

.travis.yml

@@ -13,6 +13,7 @@ branches:
   only:
   - master
   - develop
+  - feat/instantsearch.js/v2
   - "/^greenkeeper-.*$/"
 sudo: false
 cache:
@@ -22,6 +23,7 @@ env:
   - CXX=g++-4.8
   - SAUCE_USERNAME=isjs
   - SAUCE_ACCESS_KEY=3a66034c-db7c-4e93-9659-f327bdb294aa
+  - ARGOS_TOKEN=b1b69870cc497dfbeb16d1865b60ab687668bd3f
   - secure: DMNHPyjuZcxIly2N/mcqDEzeJ7gw5i6hOX+2dN8DW8CF6CFh2HIhA5Iznaerjst+gcHHRACWHT8BYstDExhaUAbegys1B/S3oDapLLT3H48a0VlpncAiyTvismv93tB7+WdoMAs+6zBMUo1z98At1ck6yUHsvASy7SNGVOI0UBRg5cu40eIrZTCKjhE6JXh2D2+HbFa7ODYy7aMl99A0Pk1SrFzehCVKd6+UhdY5Q+OKZNbPF3oDBifbAJeV54uYBPfgJJkj2CvWIIyGM33FRwrxZ/njwNgeCR1YMXNL0DEYQbyS3bBSkc/nSjE49vk8fY9m9n+CXjXsUsVWbGBBgsQTgOEQFjruNCiojXNuxTU3KLWjfDsdvT7KTnNj4lc1kfH10ESpVhhhfYLwmi9VaGyUlP825LYsdHidAWYVyLOcYufvUwXF+icpr2mEq9lG01pVZcX//GxsmQIcQ3/h3XEYfCBub7ZoiRYjoqL+AQWN+16OOW5DyC55DEI4FZClXtYP28u2z4VSpmPvE7X8kHO7fRtL5ZJyHcz9D3uVE8KlgBwtN4tOZ1tVJ8ssnaDeFJLtXpIAB/LwFdMbHNla+6gJ8aFvDiN2T97Y/Hk2/g4jO8pW1TAppnuON5pxsHy2bT46df1ad1OyINFsjtBYIQymlmVBAWalN242eDMyC+o=
   - secure: IReZG9KWjW8v+u/YmzGF8b+7zO0BC+cuvQGehxOXePnx01Ja+xrKeBv/RfO2E13vm3IoHByisPkeTWlnJA8h4/AwmvdOk33KkRBUVBABIsbvlN+uZ+jjz96K2VcD1ZjItSo7SrPEs+InRtsUfDBcr7burWUxL6tuhKcZT7wN+CT/QXBRN4Ai1z4DMFi5nK2vgOQ9u02IWjdzSXafDw4lhO+SEqNfMTDVXVqiSQ11Cyh7qsciiPkwxPhZWB3vg6YD0vU3GiYlgrC/Cr6hiCwiPi+9e+73MBwxsX8BYC0/2uM8vSMOocaePg1X9TOAP9sDCHW/MR8p+zfapV+7e7QBW2XaDy0j6m1rer1KwAhLSqCg4Gtp8Uc6qOS+W8NLnXzcUikLhkhp6/6TixsO2laKlr8sSwJ3o/qMyE7ViTyWQ/0qKZtYYFBxihT7bDzQI6C5nv+x8PZ5mFuhjaEb8PxYmgyXWznCnouqhYMHDn5geEzM3E3xjOhaIyqfNsqTX+nAkJGQR//GtZdyscKHsMD+7XVJ0AN+xRNtaEMCqVd5Tos3RT9ZSpD6eznltqL5RmASXcrKqLbuqh8ZkfpC549qvRDBFV4kOrZiiIb00H4ZcEH6WJjvRJ73EDb590HGIcsPqKlslXpRY1AhjK5AdiDnLZVrvayg+tGLFEL+8iC8q+A=
   - secure: c/zfrQ0nmOJYEx6zD+7KW0UgwQ1nlpAh1lubH+nSJwD3J3xdLi9Tm9XTy4iEiAXboxyaELgVPKaVG8kP+PzwjKfPx7a/sEYCKpkbsiTgQF4r0ldBHRX7eujbUVlYVACexBEgysDcQdg53TvuWuhctU9wk9kE9gjMFVzL61I+o7J+5wRhkhPW8A/t0eQnuqEuI9ZrYabZ6Nk1HSUidB7XGlTHeIHEQWtvCkq3Q25gOauhUeCik2ThCBlwDmvjIwkraQeKJHjd+AtmRczvfml9qh+XpT1fR9mP4nvaoiAzvjg6XsxbTlzx2D4wyI5FQNK+lpYJ/c3SfGpdIiuhE3yp8UAzkElTagFInOWLpD8hnKkbVJSUwFRW5qj/br6qyMUWKG23ItPPuRbbvR2x5zkhe2emRFtkvlQrRUZUgkDLWFh3o27t2cup2vrSGIeBUFN/TdwfQ67+x+am5h3mftzMWqCIcPdt/+ucq2xZ3Bhs+cYgjLZ6WkVkN+12Sq3q3TIICa7SDhhk3Uanz0+CuiyfK0fZd8ieUpzjNWQMxR1n8f+1EcfYNNNR71bS4sERy7HGovmemOKbYyBWx1/b2Mv/At2QlbVW1bTx/ry0xiETXf0aVETdLaVyLLBRvtj7AvqkGKP1QiwJCRa6lAhYZAeQ0Jl/9VzAII2RafiMTLUtLwA=

functional-tests/config.js

@@ -1,7 +1,17 @@
+import path from 'path';
+import {SaveScreenshot} from 'wdio-visual-regression-service/compare';
 import testServer from './testServer.js';
 import {clearAll, searchBox} from './utils.js';
 const INDEX_PAGE = process.env.INDEX_PAGE || 'index';
 
+function screenshotName(context) {
+  const testName = context.test.title.replace(/ /g, '_');
+  const name = context.browser.name.toLocaleLowerCase().replace(/ /g, '_');
+  const {width, height} = context.meta.viewport;
+
+  return path.join(__dirname, 'screenshots', `${testName}_${name}_${width}x${height}.png`);
+}
+
 let conf = {
   specs: [
     './functional-tests/test.js',
@@ -14,6 +24,15 @@ let conf = {
     compilers: ['js:babel-core/register'],
   },
   baseUrl: `http://${process.env.CI === 'true' ? 'localhost' : '10.200.10.1'}:9000`,
+  services: [
+    'visual-regression',
+  ],
+  visualRegression: {
+    compare: new SaveScreenshot({
+      screenshotName,
+    }),
+    viewportChangePause: 300, // ms
+  },
   onPrepare() {
     return testServer.start();
   },
@@ -37,7 +56,11 @@ let conf = {
 
 if (process.env.CI === 'true') {
   conf = {
-    services: ['sauce'],
+    ...conf,
+    services: [
+      ...conf.services,
+      'sauce',
+    ],
     user: process.env.SAUCE_USERNAME,
     key: process.env.SAUCE_ACCESS_KEY,
     maxInstances: 5,
@@ -84,7 +107,6 @@ if (process.env.CI === 'true') {
       //   platformName: 'iOS'
       // },
     ],
-    ...conf,
   };
 } else {
   conf = {

functional-tests/test.js

@@ -7,14 +7,24 @@ describe('searchBox', () => {
 
     it('input is empty', () => expect(searchBox.get()).toBe(''));
 
-    it('triggers an empty search', () => expect(browser.getText('#hits')).toNotContain('MP3'));
+    it('triggers an empty search', () => {
+      expect(browser.getText('#hits')).toNotContain('MP3');
+      browser.checkDocument({
+        hide: ['.ais-stats--body'], // Flaky X ms information.
+      });
+    });
   });
 
   describe('when there is a query', () => {
     beforeEach(() => searchBox.set('mp3'));
 
     it('fills the input', () => expect(searchBox.get()).toBe('mp3'));
 
-    it('triggers a new search', () => expect(browser.getText('#hits')).toContain('MP3'));
+    it('triggers a new search', () => {
+      expect(browser.getText('#hits')).toContain('MP3');
+      browser.checkDocument({
+        hide: ['.ais-stats--body'], // Flaky X ms information.
+      });
+    });
   });
 });

package.json

@@ -24,6 +24,7 @@
     "test:functional": "wdio functional-tests/wdio.conf.js",
     "test:functional:dev": "babel-node scripts/dev-functional-tests",
     "test:functional:dev:debug-server": "PORT=9090 babel-node scripts/dev-functional-tests-debug-server",
+    "argos": "argos upload functional-tests/screenshots --token $ARGOS_TOKEN || true",
     "serve": "./scripts/serve.sh"
   },
   "repository": "algolia/instantsearch.js",
@@ -35,6 +36,7 @@
     "node": "7.10.0"
   },
   "devDependencies": {
+    "argos-cli": "^0.0.8",
     "autoprefixer": "^6.7.7",
     "babel-cli": "^6.24.0",
     "babel-core": "^6.24.0",
@@ -88,6 +90,7 @@
     "wdio-dot-reporter": "^0.0.8",
     "wdio-mocha-framework": "^0.5.10",
     "wdio-sauce-service": "^0.4.0",
+    "wdio-visual-regression-service": "^0.8.0",
     "webdriverio": "^4.8.0",
     "webpack": "^1.14.0",
     "webpack-dev-server": "^1.16.2"

scripts/dev-functional-tests-compile-watch.js

@@ -17,10 +17,16 @@ export default cb => {
   // first call triggers a watch, but we already have webpack watch triggering
   // so we ignore first call
   watch.watchTree(join(__dirname, '..', 'functional-tests'), (f, curr, prev) => {
+    // Finished walking the tree
     if (typeof f === 'object' && prev === null && curr === null) {
       return;
     }
 
+    // As the ignoreDirectoryPattern option is not working
+    if (f.match(/functional-tests\/screenshots/) !== null) {
+      return;
+    }
+
     console.log('Got test file change');
     cb();
   });

scripts/dev-functional-tests.js

@@ -1,10 +1,10 @@
-/* eslint-disable no-console, no-magic-numbers */
+/* eslint-disable no-console */
 
+import {spawn} from 'child_process';
 import debounce from 'lodash/debounce';
 import watch from './dev-functional-tests-compile-watch.js';
 
-import {spawn} from 'child_process';
-
+const REBUILD_TIMEOUT = 1500;
 let wdio;
 const launch = debounce(() => {
   if (wdio) {
@@ -13,7 +13,7 @@ const launch = debounce(() => {
     wdio.kill('SIGINT'); // we need two of them, that's the way wdio works for killing process
   }
   wdio = spawn('wdio', ['functional-tests/wdio.conf.js'], {stdio: [null, process.stdout, null]});
-}, 1500, {
+}, REBUILD_TIMEOUT, {
   leading: true,
   trailing: true,
 });

scripts/test-ci.sh

@@ -10,8 +10,8 @@ NODE_ENV=production npm run build
 INDEX_PAGE=index npm run test:functional
 ./scripts/validate-commit-msgs.sh
 
-# check we are able to generate docs
-npm run docs:jsdoc
+# push the generated screenshots to Argos-CI.
+npm run argos
 
 if [ $TRAVIS_PULL_REQUEST == 'false' ] && [ $TRAVIS_BRANCH == 'master' ]; then
   npm run finish-release

scripts/validate-pr-done-on-develop.sh

@@ -7,7 +7,6 @@ COMMIT_MSG=$(git log --format=%B --no-merges -n 1)
 [[ "$COMMIT_MSG" =~ ^docs? ]] && is_doc=1 || is_doc=0
 [[ "$TRAVIS_PULL_REQUEST" == false ]] && is_pr=0 || is_pr=1
 [[ "$TRAVIS_BRANCH" == master ]] && is_master=1 || is_master=0
-[[ "$TRAVIS_BRANCH" == develop ]] && is_develop=1 || is_develop=0
 
 if [[ $is_pr == 0 ]]; then
   echo "Success: This is not a pull request"
@@ -28,7 +27,7 @@ if [[ $is_master == 1 ]]; then
   exit 1
 fi
 
-if [[ $is_develop == 0 ]]; then
+if [[ $is_master == 1 ]]; then
   echo "Error: Pull request must be done on develop branch"
   exit 1
 fi