maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #7529 from cjcenizal/7523/visual-regression-visualize

Browse source 
Create visual_regression test suite.
This commit is contained in:
CJ Cenizal 2016-06-22 07:22:56 -07:00 committed by GitHub
parent ded0ee0a03 37f82f146f
commit 9a242cf8f5
12 changed files with 430 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -48,6 +48,7 @@
"test:ui:runner": "grunt test:ui:runner",
"test:server": "grunt test:server",
"test:coverage": "grunt test:coverage",
"test:visualRegression": "grunt test:visualRegression",
"build": "grunt build",
"release": "grunt release",
"start": "sh ./bin/kibana --dev",

8
tasks/config/intern.js
View file

@ -13,7 +13,13 @@ module.exports = function (grunt) {
api: {
options: {
runType: 'client',
config: 'test/api_intern'
config: 'test/intern_api'
}
},
visualRegression: {
options: {
runType: 'runner',
config: 'test/intern_visual_regression'
}
}
};

15
tasks/test.js
View file

@ -2,8 +2,18 @@ const _ = require('lodash');
const visualRegression = require('../utilities/visual_regression');
module.exports = function (grunt) {
grunt.registerTask('test:visualRegression', [
'intern:visualRegression:takeScreenshots',
'test:visualRegression:buildGallery'
]);
grunt.registerTask('test:visualRegression:takeScreenshots', [
'clean:screenshots',
'intern:visualRegression'
]);
grunt.registerTask(
'test:visualRegression',
'test:visualRegression:buildGallery',
'Compare screenshots and generate diff images.',
function () {
const done = this.async();
@ -72,8 +82,7 @@ module.exports = function (grunt) {
grunt.task.run(_.compact([
!grunt.option('quick') && 'eslint:source',
'licenses',
'test:quick',
'test:visualRegression'
'test:quick'
]));
});

12
test/functional/index.js
View file

@ -3,18 +3,18 @@ define(function (require) {
const bdd = require('intern!bdd');
const intern = require('intern');
const earliestBeforeCbs = [];
const initCallbacks = [];
function onEarliestBefore(cb) {
earliestBeforeCbs.push(cb);
function onInit(callback) {
initCallbacks.push(callback);
}
global.__kibana__intern__ = { intern, bdd, onEarliestBefore };
global.__kibana__intern__ = { intern, bdd, onInit };
bdd.describe('kibana', function () {
bdd.before(function () {
earliestBeforeCbs.forEach(cb => {
cb.call(this);
initCallbacks.forEach(callback => {
callback.call(this);
});
});

6
test/intern.js
View file

@ -1,8 +1,6 @@
define(function (require) {
var serverConfig = require('intern/dojo/node!./server_config');
var _ = require('intern/dojo/node!lodash');
return _.assign({
const serverConfig = require('intern/dojo/node!./server_config');
return Object.assign({
debug: true,
capabilities: {
'selenium-version': '2.53.0',

0
test/api_intern.js → test/intern_api.js
View file

24
test/intern_visual_regression.js Normal file
View file

@ -0,0 +1,24 @@
define(function (require) {
const serverConfig = require('intern/dojo/node!./server_config');
return Object.assign({
debug: true,
capabilities: {
'selenium-version': '2.53.0',
// must match URL in tasks/config/downloadSelenium.js
'idle-timeout': 99
},
environments: [{
browserName: 'chrome'
}],
tunnelOptions: serverConfig.servers.webdriver,
functionalSuites: [
'test/visual_regression/index'
],
excludeInstrumentation: /.*/,
defaultTimeout: 90000,
defaultTryTimeout: 40000, // tryForTime could include multiple 'find timeouts'
defaultFindTimeout: 10000 // this is how long we try to find elements on page
}, serverConfig);
});

62
test/support/index.js
View file

@ -23,36 +23,64 @@ exports.scenarioManager = new ScenarioManager(url.format(exports.config.servers.
exports.esClient = new EsClient(url.format(exports.config.servers.elasticsearch));
exports.bdd = new BddWrapper(kbnInternVars.bdd);
defineDelayedExport('remote', (suite) => suite.remote);
defineDelayedExport('common', () => new Common());
defineDelayedExport('discoverPage', () => new DiscoverPage());
defineDelayedExport('headerPage', () => new HeaderPage());
defineDelayedExport('settingsPage', () => new SettingsPage());
defineDelayedExport('visualizePage', () => new VisualizePage());
defineDelayedExport('dashboardPage', () => new DashboardPage());
defineDelayedExport('shieldPage', () => new ShieldPage());
defineDelayedExport('consolePage', () => new ConsolePage());
defineDelayedExport('elasticDump', () => new ElasticDump());
const delayedExports = [{
name: 'remote',
exportProvider: (suite) => suite.remote
}, {
name: 'common',
exportProvider: () => new Common()
}, {
name: 'discoverPage',
exportProvider: () => new DiscoverPage()
}, {
name: 'headerPage',
exportProvider: () => new HeaderPage()
}, {
name: 'settingsPage',
exportProvider: () => new SettingsPage()
}, {
name: 'visualizePage',
exportProvider: () => new VisualizePage()
}, {
name: 'dashboardPage',
exportProvider: () => new DashboardPage()
}, {
name: 'shieldPage',
exportProvider: () => new ShieldPage()
}, {
name: 'consolePage',
exportProvider: () => new ConsolePage()
}, {
name: 'elasticDump',
exportProvider: () => new ElasticDump()
}];
// creates an export for values that aren't actually avaialable until
// until tests start to run. These getters will throw errors if the export
delayedExports.forEach(exportConfig => {
delayExportUntilInit(
exportConfig.name,
exportConfig.exportProvider
);
});
// Creates an export for values that aren't actually avaialable until
// tests start to run. These getters will throw errors if the export
// is accessed before it's available, hopefully making debugging easier.
// Once the first before() handler is called the onEarliestBefore() handlers
// Once the first before() handler is called the onInit() call
// will fire and rewrite all of these exports to be their correct value.
function defineDelayedExport(name, define) {
function delayExportUntilInit(name, provideExport) {
Object.defineProperty(exports, name, {
configurable: true,
get() {
throw new TypeError(
'remote is not available until tests start to run. Move your ' +
'Remote is not available until tests start to run. Move your ' +
'usage of the import inside a test setup hook or a test itself.'
);
}
});
kbnInternVars.onEarliestBefore(function () {
kbnInternVars.onInit(function defineExport() {
Object.defineProperty(exports, name, {
value: define(this),
value: provideExport(this),
});
});
}

550
test/support/pages/common.js
View file

@ -1,48 +1,58 @@
import { config, defaultTryTimeout, defaultFindTimeout, remote, shieldPage } from '../';
import bluebird, {
promisify
} from 'bluebird';
import fs from 'fs';
import _ from 'lodash';
import mkdirp from 'mkdirp';
import { promisify } from 'bluebird';
import moment from 'moment';
import path from 'path';
import testSubjSelector from '@spalger/test-subj-selector';
import {
format,
parse
} from 'url';
import util from 'util';
import getUrl from '../../utils/get_url';
import {
config,
defaultTryTimeout,
defaultFindTimeout,
remote,
shieldPage
} from '../index';
const mkdirpAsync = promisify(mkdirp);
const writeFileAsync = promisify(fs.writeFile);
export default (function () {
var Promise = require('bluebird');
var moment = require('moment');
var testSubjSelector = require('@spalger/test-subj-selector');
var getUrl = require('../../utils/get_url');
var _ = require('lodash');
var parse = require('url').parse;
var format = require('url').format;
var util = require('util');
var path = require('path');
export default class Common {
function injectTimestampQuery(func, url) {
var formatted = modifyQueryString(url, function (parsed) {
parsed.query._t = Date.now();
});
return func.call(this, formatted);
}
function removeTimestampQuery(func) {
return func.call(this)
.then(function (url) {
return modifyQueryString(url, function (parsed) {
parsed.query = _.omit(parsed.query, '_t');
constructor() {
function injectTimestampQuery(func, url) {
var formatted = modifyQueryString(url, function (parsed) {
parsed.query._t = Date.now();
});
});
}
function modifyQueryString(url, func) {
var parsed = parse(url, true);
if (parsed.query === null) {
parsed.query = {};
return func.call(this, formatted);
}
function removeTimestampQuery(func) {
return func.call(this)
.then(function (url) {
return modifyQueryString(url, function (parsed) {
parsed.query = _.omit(parsed.query, '_t');
});
});
}
function modifyQueryString(url, func) {
var parsed = parse(url, true);
if (parsed.query === null) {
parsed.query = {};
}
func(parsed);
return format(_.pick(parsed, 'protocol', 'hostname', 'port', 'pathname', 'query', 'hash', 'auth'));
}
func(parsed);
return format(_.pick(parsed, 'protocol', 'hostname', 'port', 'pathname', 'query', 'hash', 'auth'));
}
function Common() {
this.remote = remote;
if (remote.get.wrapper !== injectTimestampQuery) {
this.remote.get = _.wrap(this.remote.get, injectTimestampQuery);
@ -51,253 +61,247 @@ export default (function () {
}
}
Common.prototype = {
constructor: Common,
getHostPort() {
return getUrl.baseUrl(config.servers.kibana);
}
getHostPort: function getHostPort() {
return getUrl.baseUrl(config.servers.kibana);
},
getEsHostPort() {
return getUrl.baseUrl(config.servers.elasticsearch);
}
getEsHostPort: function getHostPort() {
return getUrl.baseUrl(config.servers.elasticsearch);
},
navigateToApp(appName, testStatusPage) {
var self = this;
var appUrl = getUrl.noAuth(config.servers.kibana, config.apps[appName]);
self.debug('navigating to ' + appName + ' url: ' + appUrl);
navigateToApp: function (appName, testStatusPage) {
var self = this;
var appUrl = getUrl.noAuth(config.servers.kibana, config.apps[appName]);
self.debug('navigating to ' + appName + ' url: ' + appUrl);
var doNavigation = function (url) {
return self.try(function () {
// since we're using hash URLs, always reload first to force re-render
self.debug('navigate to: ' + url);
return self.remote.get(url)
.then(function () {
return self.sleep(700);
})
.then(function () {
self.debug('returned from get, calling refresh');
return self.remote.refresh();
})
.then(function () {
self.debug('check testStatusPage');
if (testStatusPage !== false) {
self.debug('self.checkForKibanaApp()');
return self.checkForKibanaApp()
.then(function (kibanaLoaded) {
self.debug('kibanaLoaded = ' + kibanaLoaded);
if (!kibanaLoaded) {
var msg = 'Kibana is not loaded, retrying';
self.debug(msg);
throw new Error(msg);
}
});
}
})
.then(function () {
return self.remote.getCurrentUrl();
})
.then(function (currentUrl) {
var loginPage = new RegExp('login').test(currentUrl);
if (loginPage) {
self.debug('Found loginPage = ' + loginPage + ', username = '
+ config.servers.kibana.shield.username);
return shieldPage.login(config.servers.kibana.shield.username,
config.servers.kibana.shield.password)
.then(function () {
return self.remote.getCurrentUrl();
});
} else {
return currentUrl;
}
})
.then(function (currentUrl) {
currentUrl = currentUrl.replace(/\/\/\w+:\w+@/, '//');
var maxAdditionalLengthOnNavUrl = 230;
// On several test failures at the end of the TileMap test we try to navigate back to
// Visualize so we can create the next Vertical Bar Chart, but we can see from the
// logging and the screenshot that it's still on the TileMap page. Why didn't the "get"
// with a new timestamped URL go? I thought that sleep(700) between the get and the
// refresh would solve the problem but didn't seem to always work.
// So this hack fails the navSuccessful check if the currentUrl doesn't match the
// appUrl plus up to 230 other chars.
// Navigating to Settings when there is a default index pattern has a URL length of 196
// (from debug output). Some other tabs may also be long. But a rather simple configured
// visualization is about 1000 chars long. So at least we catch that case.
var navSuccessful = new RegExp(appUrl + '.{0,' + maxAdditionalLengthOnNavUrl + '}$')
.test(currentUrl);
if (!navSuccessful) {
var msg = 'App failed to load: ' + appName +
' in ' + defaultFindTimeout + 'ms' +
' appUrl = ' + appUrl +
' currentUrl = ' + currentUrl;
self.debug(msg);
throw new Error(msg);
}
return currentUrl;
});
});
};
return doNavigation(appUrl)
.then(function (currentUrl) {
var lastUrl = currentUrl;
return self.try(function () {
// give the app time to update the URL
return self.sleep(501)
.then(function () {
return self.remote.getCurrentUrl();
})
.then(function (currentUrl) {
self.debug('in doNavigation url = ' + currentUrl);
if (lastUrl !== currentUrl) {
lastUrl = currentUrl;
throw new Error('URL changed, waiting for it to settle');
}
});
});
});
},
runScript: function (fn, timeout) {
var self = this;
// by default, give the app 10 seconds to load
timeout = timeout || 10000;
// wait for deps on window before running script
return self.remote
.setExecuteAsyncTimeout(timeout)
.executeAsync(function (done) {
var interval = setInterval(function () {
var ready = (document.readyState === 'complete');
var hasJQuery = !!window.$;
if (ready && hasJQuery) {
console.log('doc ready, jquery loaded');
clearInterval(interval);
done();
}
}, 10);
}).then(function () {
return self.remote.execute(fn);
});
},
getApp: function () {
var self = this;
return self.remote.setFindTimeout(defaultFindTimeout)
.findByCssSelector('.app-wrapper .application')
.then(function () {
return self.runScript(function () {
var $ = window.$;
var $scope = $('.app-wrapper .application').scope();
return $scope ? $scope.chrome.getApp() : {};
});
});
},
checkForKibanaApp: function () {
var self = this;
return self.getApp()
.then(function (app) {
var appId = app.id;
self.debug('current application: ' + appId);
return appId === 'kibana';
})
.catch(function (err) {
self.debug('kibana check failed');
self.debug(err);
// not on the kibana app...
return false;
});
},
tryForTime: function (timeout, block) {
var self = this;
var start = Date.now();
var retryDelay = 502;
var lastTry = 0;
var tempMessage;
function attempt() {
lastTry = Date.now();
if (lastTry - start > timeout) {
throw new Error('timeout ' + tempMessage);
}
return Promise
.try(block)
.catch(function tryForTimeCatch(err) {
self.debug('tryForTime failure: ' + err.message);
tempMessage = err.message;
return Promise.delay(retryDelay).then(attempt);
});
}
return Promise.try(attempt);
},
try(block) {
return this.tryForTime(defaultTryTimeout, block);
},
log(...args) {
console.log(moment().format('HH:mm:ss.SSS') + ':', util.format(...args));
},
debug(...args) {
if (config.debug) this.log(...args);
},
sleep: function sleep(sleepMilliseconds) {
var self = this;
self.debug('... sleep(' + sleepMilliseconds + ') start');
return Promise.resolve().delay(sleepMilliseconds)
.then(function () { self.debug('... sleep(' + sleepMilliseconds + ') end'); });
},
handleError(testObj) {
const testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
return reason => {
const now = Date.now();
const fileName = `failure_${now}_${testName}`;
return this.saveScreenshot(fileName, true)
function navigateTo(url) {
return self.try(function () {
// since we're using hash URLs, always reload first to force re-render
self.debug('navigate to: ' + url);
return self.remote.get(url)
.then(function () {
throw reason;
return self.sleep(700);
})
.then(function () {
self.debug('returned from get, calling refresh');
return self.remote.refresh();
})
.then(function () {
self.debug('check testStatusPage');
if (testStatusPage !== false) {
self.debug('self.checkForKibanaApp()');
return self.checkForKibanaApp()
.then(function (kibanaLoaded) {
self.debug('kibanaLoaded = ' + kibanaLoaded);
if (!kibanaLoaded) {
var msg = 'Kibana is not loaded, retrying';
self.debug(msg);
throw new Error(msg);
}
});
}
})
.then(function () {
return self.remote.getCurrentUrl();
})
.then(function (currentUrl) {
var loginPage = new RegExp('login').test(currentUrl);
if (loginPage) {
self.debug('Found loginPage = ' + loginPage + ', username = '
+ config.servers.kibana.shield.username);
return shieldPage.login(config.servers.kibana.shield.username,
config.servers.kibana.shield.password)
.then(function () {
return self.remote.getCurrentUrl();
});
} else {
return currentUrl;
}
})
.then(function (currentUrl) {
currentUrl = currentUrl.replace(/\/\/\w+:\w+@/, '//');
var maxAdditionalLengthOnNavUrl = 230;
// On several test failures at the end of the TileMap test we try to navigate back to
// Visualize so we can create the next Vertical Bar Chart, but we can see from the
// logging and the screenshot that it's still on the TileMap page. Why didn't the "get"
// with a new timestamped URL go? I thought that sleep(700) between the get and the
// refresh would solve the problem but didn't seem to always work.
// So this hack fails the navSuccessful check if the currentUrl doesn't match the
// appUrl plus up to 230 other chars.
// Navigating to Settings when there is a default index pattern has a URL length of 196
// (from debug output). Some other tabs may also be long. But a rather simple configured
// visualization is about 1000 chars long. So at least we catch that case.
var navSuccessful = new RegExp(appUrl + '.{0,' + maxAdditionalLengthOnNavUrl + '}$')
.test(currentUrl);
if (!navSuccessful) {
var msg = 'App failed to load: ' + appName +
' in ' + defaultFindTimeout + 'ms' +
' appUrl = ' + appUrl +
' currentUrl = ' + currentUrl;
self.debug(msg);
throw new Error(msg);
}
return currentUrl;
});
};
},
});
};
async saveScreenshot(fileName, isFailure = false) {
try {
const directoryName = isFailure ? 'failure' : 'session';
const directoryPath = path.resolve(`test/screenshots/${directoryName}`);
const filePath = path.resolve(directoryPath, `${fileName}.png`);
this.debug(`Taking screenshot "${filePath}"`);
return navigateTo(appUrl)
.then(function (currentUrl) {
var lastUrl = currentUrl;
return self.try(function () {
// give the app time to update the URL
return self.sleep(501)
.then(function () {
return self.remote.getCurrentUrl();
})
.then(function (currentUrl) {
self.debug('in navigateTo url = ' + currentUrl);
if (lastUrl !== currentUrl) {
lastUrl = currentUrl;
throw new Error('URL changed, waiting for it to settle');
}
});
});
});
}
const screenshot = await this.remote.takeScreenshot();
await mkdirpAsync(directoryPath);
await writeFileAsync(filePath, screenshot);
} catch (err) {
this.log(`SCREENSHOT FAILED: ${err}`);
runScript(fn, timeout) {
var self = this;
// by default, give the app 10 seconds to load
timeout = timeout || 10000;
// wait for deps on window before running script
return self.remote
.setExecuteAsyncTimeout(timeout)
.executeAsync(function (done) {
var interval = setInterval(function () {
var ready = (document.readyState === 'complete');
var hasJQuery = !!window.$;
if (ready && hasJQuery) {
console.log('doc ready, jquery loaded');
clearInterval(interval);
done();
}
}, 10);
}).then(function () {
return self.remote.execute(fn);
});
}
getApp() {
var self = this;
return self.remote.setFindTimeout(defaultFindTimeout)
.findByCssSelector('.app-wrapper .application')
.then(function () {
return self.runScript(function () {
var $ = window.$;
var $scope = $('.app-wrapper .application').scope();
return $scope ? $scope.chrome.getApp() : {};
});
});
}
checkForKibanaApp() {
var self = this;
return self.getApp()
.then(function (app) {
var appId = app.id;
self.debug('current application: ' + appId);
return appId === 'kibana';
})
.catch(function (err) {
self.debug('kibana check failed');
self.debug(err);
// not on the kibana app...
return false;
});
}
tryForTime(timeout, block) {
var self = this;
var start = Date.now();
var retryDelay = 502;
var lastTry = 0;
var tempMessage;
function attempt() {
lastTry = Date.now();
if (lastTry - start > timeout) {
throw new Error('timeout ' + tempMessage);
}
},
findTestSubject: function findTestSubject(selector) {
this.debug('in findTestSubject: ' + testSubjSelector(selector));
return this.remote
.setFindTimeout(defaultFindTimeout)
.findDisplayedByCssSelector(testSubjSelector(selector));
return bluebird
.try(block)
.catch(function tryForTimeCatch(err) {
self.debug('tryForTime failure: ' + err.message);
tempMessage = err.message;
return bluebird.delay(retryDelay).then(attempt);
});
}
};
return bluebird.try(attempt);
}
return Common;
}());
try(block) {
return this.tryForTime(defaultTryTimeout, block);
}
log(...args) {
console.log(moment().format('HH:mm:ss.SSS') + ':', util.format(...args));
}
debug(...args) {
if (config.debug) this.log(...args);
}
sleep(sleepMilliseconds) {
var self = this;
self.debug('... sleep(' + sleepMilliseconds + ') start');
return bluebird.resolve().delay(sleepMilliseconds)
.then(function () { self.debug('... sleep(' + sleepMilliseconds + ') end'); });
}
handleError(testObj) {
const testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
return reason => {
const now = Date.now();
const fileName = `failure_${now}_${testName}`;
return this.saveScreenshot(fileName, true)
.then(function () {
throw reason;
});
};
}
async saveScreenshot(fileName, isFailure = false) {
try {
const directoryName = isFailure ? 'failure' : 'session';
const directoryPath = path.resolve(`test/screenshots/${directoryName}`);
const filePath = path.resolve(directoryPath, `${fileName}.png`);
this.debug(`Taking screenshot "${filePath}"`);
const screenshot = await this.remote.takeScreenshot();
await mkdirpAsync(directoryPath);
await writeFileAsync(filePath, screenshot);
} catch (err) {
this.log(`SCREENSHOT FAILED: ${err}`);
}
}
findTestSubject(selector) {
this.debug('in findTestSubject: ' + testSubjSelector(selector));
return this.remote
.setFindTimeout(defaultFindTimeout)
.findDisplayedByCssSelector(testSubjSelector(selector));
}
};

14
test/visual_regression/home/_loading.js Normal file
View file

@ -0,0 +1,14 @@
import {
bdd,
scenarioManager,
common,
consolePage
} from '../../support';
var expect = require('expect.js');
bdd.describe('Loading', function coverLoadingUi() {
bdd.it('should show loading feebdack', async function () {
// TODO: Take screenshots here.
});
});

16
test/visual_regression/home/index.js Normal file
View file

@ -0,0 +1,16 @@
import {
bdd,
remote,
scenarioManager,
defaultTimeout
} from '../../support';
bdd.describe('Home', function () {
this.timeout = defaultTimeout;
bdd.before(function () {
return remote.setWindowSize(1200, 800);
});
require('./_loading');
});

26
test/visual_regression/index.js Normal file
View file

@ -0,0 +1,26 @@
define(function (require) {
require('intern/dojo/node!../support/env_setup');
const bdd = require('intern!bdd');
const intern = require('intern');
const initCallbacks = [];
function onInit(callback) {
initCallbacks.push(callback);
}
global.__kibana__intern__ = { intern, bdd, onInit };
bdd.describe('Kibana visual regressions', function () {
bdd.before(function () {
initCallbacks.forEach(callback => {
callback.call(this);
});
});
require([
'intern/dojo/node!../support/index',
'intern/dojo/node!./home',
], function () {});
});
});