Merge pull request #7529 from cjcenizal/7523/visual-regression-visualize
Create visual_regression test suite.
This commit is contained in:
commit
9a242cf8f5
|
@ -48,6 +48,7 @@
|
|||
"test:ui:runner": "grunt test:ui:runner",
|
||||
"test:server": "grunt test:server",
|
||||
"test:coverage": "grunt test:coverage",
|
||||
"test:visualRegression": "grunt test:visualRegression",
|
||||
"build": "grunt build",
|
||||
"release": "grunt release",
|
||||
"start": "sh ./bin/kibana --dev",
|
||||
|
|
|
@ -13,7 +13,13 @@ module.exports = function (grunt) {
|
|||
api: {
|
||||
options: {
|
||||
runType: 'client',
|
||||
config: 'test/api_intern'
|
||||
config: 'test/intern_api'
|
||||
}
|
||||
},
|
||||
visualRegression: {
|
||||
options: {
|
||||
runType: 'runner',
|
||||
config: 'test/intern_visual_regression'
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
|
@ -2,8 +2,18 @@ const _ = require('lodash');
|
|||
const visualRegression = require('../utilities/visual_regression');
|
||||
|
||||
module.exports = function (grunt) {
|
||||
grunt.registerTask('test:visualRegression', [
|
||||
'intern:visualRegression:takeScreenshots',
|
||||
'test:visualRegression:buildGallery'
|
||||
]);
|
||||
|
||||
grunt.registerTask('test:visualRegression:takeScreenshots', [
|
||||
'clean:screenshots',
|
||||
'intern:visualRegression'
|
||||
]);
|
||||
|
||||
grunt.registerTask(
|
||||
'test:visualRegression',
|
||||
'test:visualRegression:buildGallery',
|
||||
'Compare screenshots and generate diff images.',
|
||||
function () {
|
||||
const done = this.async();
|
||||
|
@ -72,8 +82,7 @@ module.exports = function (grunt) {
|
|||
grunt.task.run(_.compact([
|
||||
!grunt.option('quick') && 'eslint:source',
|
||||
'licenses',
|
||||
'test:quick',
|
||||
'test:visualRegression'
|
||||
'test:quick'
|
||||
]));
|
||||
});
|
||||
|
||||
|
|
|
@ -3,18 +3,18 @@ define(function (require) {
|
|||
|
||||
const bdd = require('intern!bdd');
|
||||
const intern = require('intern');
|
||||
const earliestBeforeCbs = [];
|
||||
const initCallbacks = [];
|
||||
|
||||
function onEarliestBefore(cb) {
|
||||
earliestBeforeCbs.push(cb);
|
||||
function onInit(callback) {
|
||||
initCallbacks.push(callback);
|
||||
}
|
||||
|
||||
global.__kibana__intern__ = { intern, bdd, onEarliestBefore };
|
||||
global.__kibana__intern__ = { intern, bdd, onInit };
|
||||
|
||||
bdd.describe('kibana', function () {
|
||||
bdd.before(function () {
|
||||
earliestBeforeCbs.forEach(cb => {
|
||||
cb.call(this);
|
||||
initCallbacks.forEach(callback => {
|
||||
callback.call(this);
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
define(function (require) {
|
||||
var serverConfig = require('intern/dojo/node!./server_config');
|
||||
var _ = require('intern/dojo/node!lodash');
|
||||
|
||||
return _.assign({
|
||||
const serverConfig = require('intern/dojo/node!./server_config');
|
||||
return Object.assign({
|
||||
debug: true,
|
||||
capabilities: {
|
||||
'selenium-version': '2.53.0',
|
||||
|
|
24
test/intern_visual_regression.js
Normal file
24
test/intern_visual_regression.js
Normal file
|
@ -0,0 +1,24 @@
|
|||
define(function (require) {
|
||||
const serverConfig = require('intern/dojo/node!./server_config');
|
||||
return Object.assign({
|
||||
debug: true,
|
||||
capabilities: {
|
||||
'selenium-version': '2.53.0',
|
||||
// must match URL in tasks/config/downloadSelenium.js
|
||||
'idle-timeout': 99
|
||||
},
|
||||
environments: [{
|
||||
browserName: 'chrome'
|
||||
}],
|
||||
tunnelOptions: serverConfig.servers.webdriver,
|
||||
functionalSuites: [
|
||||
'test/visual_regression/index'
|
||||
],
|
||||
|
||||
excludeInstrumentation: /.*/,
|
||||
|
||||
defaultTimeout: 90000,
|
||||
defaultTryTimeout: 40000, // tryForTime could include multiple 'find timeouts'
|
||||
defaultFindTimeout: 10000 // this is how long we try to find elements on page
|
||||
}, serverConfig);
|
||||
});
|
|
@ -23,36 +23,64 @@ exports.scenarioManager = new ScenarioManager(url.format(exports.config.servers.
|
|||
exports.esClient = new EsClient(url.format(exports.config.servers.elasticsearch));
|
||||
exports.bdd = new BddWrapper(kbnInternVars.bdd);
|
||||
|
||||
defineDelayedExport('remote', (suite) => suite.remote);
|
||||
defineDelayedExport('common', () => new Common());
|
||||
defineDelayedExport('discoverPage', () => new DiscoverPage());
|
||||
defineDelayedExport('headerPage', () => new HeaderPage());
|
||||
defineDelayedExport('settingsPage', () => new SettingsPage());
|
||||
defineDelayedExport('visualizePage', () => new VisualizePage());
|
||||
defineDelayedExport('dashboardPage', () => new DashboardPage());
|
||||
defineDelayedExport('shieldPage', () => new ShieldPage());
|
||||
defineDelayedExport('consolePage', () => new ConsolePage());
|
||||
defineDelayedExport('elasticDump', () => new ElasticDump());
|
||||
const delayedExports = [{
|
||||
name: 'remote',
|
||||
exportProvider: (suite) => suite.remote
|
||||
}, {
|
||||
name: 'common',
|
||||
exportProvider: () => new Common()
|
||||
}, {
|
||||
name: 'discoverPage',
|
||||
exportProvider: () => new DiscoverPage()
|
||||
}, {
|
||||
name: 'headerPage',
|
||||
exportProvider: () => new HeaderPage()
|
||||
}, {
|
||||
name: 'settingsPage',
|
||||
exportProvider: () => new SettingsPage()
|
||||
}, {
|
||||
name: 'visualizePage',
|
||||
exportProvider: () => new VisualizePage()
|
||||
}, {
|
||||
name: 'dashboardPage',
|
||||
exportProvider: () => new DashboardPage()
|
||||
}, {
|
||||
name: 'shieldPage',
|
||||
exportProvider: () => new ShieldPage()
|
||||
}, {
|
||||
name: 'consolePage',
|
||||
exportProvider: () => new ConsolePage()
|
||||
}, {
|
||||
name: 'elasticDump',
|
||||
exportProvider: () => new ElasticDump()
|
||||
}];
|
||||
|
||||
// creates an export for values that aren't actually avaialable until
|
||||
// until tests start to run. These getters will throw errors if the export
|
||||
delayedExports.forEach(exportConfig => {
|
||||
delayExportUntilInit(
|
||||
exportConfig.name,
|
||||
exportConfig.exportProvider
|
||||
);
|
||||
});
|
||||
|
||||
// Creates an export for values that aren't actually avaialable until
|
||||
// tests start to run. These getters will throw errors if the export
|
||||
// is accessed before it's available, hopefully making debugging easier.
|
||||
// Once the first before() handler is called the onEarliestBefore() handlers
|
||||
// Once the first before() handler is called the onInit() call
|
||||
// will fire and rewrite all of these exports to be their correct value.
|
||||
function defineDelayedExport(name, define) {
|
||||
function delayExportUntilInit(name, provideExport) {
|
||||
Object.defineProperty(exports, name, {
|
||||
configurable: true,
|
||||
get() {
|
||||
throw new TypeError(
|
||||
'remote is not available until tests start to run. Move your ' +
|
||||
'Remote is not available until tests start to run. Move your ' +
|
||||
'usage of the import inside a test setup hook or a test itself.'
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
kbnInternVars.onEarliestBefore(function () {
|
||||
kbnInternVars.onInit(function defineExport() {
|
||||
Object.defineProperty(exports, name, {
|
||||
value: define(this),
|
||||
value: provideExport(this),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
@ -1,48 +1,58 @@
|
|||
import { config, defaultTryTimeout, defaultFindTimeout, remote, shieldPage } from '../';
|
||||
import bluebird, {
|
||||
promisify
|
||||
} from 'bluebird';
|
||||
import fs from 'fs';
|
||||
import _ from 'lodash';
|
||||
import mkdirp from 'mkdirp';
|
||||
import { promisify } from 'bluebird';
|
||||
import moment from 'moment';
|
||||
import path from 'path';
|
||||
import testSubjSelector from '@spalger/test-subj-selector';
|
||||
import {
|
||||
format,
|
||||
parse
|
||||
} from 'url';
|
||||
import util from 'util';
|
||||
|
||||
import getUrl from '../../utils/get_url';
|
||||
import {
|
||||
config,
|
||||
defaultTryTimeout,
|
||||
defaultFindTimeout,
|
||||
remote,
|
||||
shieldPage
|
||||
} from '../index';
|
||||
|
||||
const mkdirpAsync = promisify(mkdirp);
|
||||
const writeFileAsync = promisify(fs.writeFile);
|
||||
|
||||
export default (function () {
|
||||
var Promise = require('bluebird');
|
||||
var moment = require('moment');
|
||||
var testSubjSelector = require('@spalger/test-subj-selector');
|
||||
var getUrl = require('../../utils/get_url');
|
||||
var _ = require('lodash');
|
||||
var parse = require('url').parse;
|
||||
var format = require('url').format;
|
||||
var util = require('util');
|
||||
var path = require('path');
|
||||
export default class Common {
|
||||
|
||||
function injectTimestampQuery(func, url) {
|
||||
var formatted = modifyQueryString(url, function (parsed) {
|
||||
parsed.query._t = Date.now();
|
||||
});
|
||||
return func.call(this, formatted);
|
||||
}
|
||||
|
||||
function removeTimestampQuery(func) {
|
||||
return func.call(this)
|
||||
.then(function (url) {
|
||||
return modifyQueryString(url, function (parsed) {
|
||||
parsed.query = _.omit(parsed.query, '_t');
|
||||
constructor() {
|
||||
function injectTimestampQuery(func, url) {
|
||||
var formatted = modifyQueryString(url, function (parsed) {
|
||||
parsed.query._t = Date.now();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function modifyQueryString(url, func) {
|
||||
var parsed = parse(url, true);
|
||||
if (parsed.query === null) {
|
||||
parsed.query = {};
|
||||
return func.call(this, formatted);
|
||||
}
|
||||
|
||||
function removeTimestampQuery(func) {
|
||||
return func.call(this)
|
||||
.then(function (url) {
|
||||
return modifyQueryString(url, function (parsed) {
|
||||
parsed.query = _.omit(parsed.query, '_t');
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function modifyQueryString(url, func) {
|
||||
var parsed = parse(url, true);
|
||||
if (parsed.query === null) {
|
||||
parsed.query = {};
|
||||
}
|
||||
func(parsed);
|
||||
return format(_.pick(parsed, 'protocol', 'hostname', 'port', 'pathname', 'query', 'hash', 'auth'));
|
||||
}
|
||||
func(parsed);
|
||||
return format(_.pick(parsed, 'protocol', 'hostname', 'port', 'pathname', 'query', 'hash', 'auth'));
|
||||
}
|
||||
|
||||
function Common() {
|
||||
this.remote = remote;
|
||||
if (remote.get.wrapper !== injectTimestampQuery) {
|
||||
this.remote.get = _.wrap(this.remote.get, injectTimestampQuery);
|
||||
|
@ -51,253 +61,247 @@ export default (function () {
|
|||
}
|
||||
}
|
||||
|
||||
Common.prototype = {
|
||||
constructor: Common,
|
||||
getHostPort() {
|
||||
return getUrl.baseUrl(config.servers.kibana);
|
||||
}
|
||||
|
||||
getHostPort: function getHostPort() {
|
||||
return getUrl.baseUrl(config.servers.kibana);
|
||||
},
|
||||
getEsHostPort() {
|
||||
return getUrl.baseUrl(config.servers.elasticsearch);
|
||||
}
|
||||
|
||||
getEsHostPort: function getHostPort() {
|
||||
return getUrl.baseUrl(config.servers.elasticsearch);
|
||||
},
|
||||
navigateToApp(appName, testStatusPage) {
|
||||
var self = this;
|
||||
var appUrl = getUrl.noAuth(config.servers.kibana, config.apps[appName]);
|
||||
self.debug('navigating to ' + appName + ' url: ' + appUrl);
|
||||
|
||||
navigateToApp: function (appName, testStatusPage) {
|
||||
var self = this;
|
||||
var appUrl = getUrl.noAuth(config.servers.kibana, config.apps[appName]);
|
||||
self.debug('navigating to ' + appName + ' url: ' + appUrl);
|
||||
|
||||
var doNavigation = function (url) {
|
||||
return self.try(function () {
|
||||
// since we're using hash URLs, always reload first to force re-render
|
||||
self.debug('navigate to: ' + url);
|
||||
return self.remote.get(url)
|
||||
.then(function () {
|
||||
return self.sleep(700);
|
||||
})
|
||||
.then(function () {
|
||||
self.debug('returned from get, calling refresh');
|
||||
return self.remote.refresh();
|
||||
})
|
||||
.then(function () {
|
||||
self.debug('check testStatusPage');
|
||||
if (testStatusPage !== false) {
|
||||
self.debug('self.checkForKibanaApp()');
|
||||
return self.checkForKibanaApp()
|
||||
.then(function (kibanaLoaded) {
|
||||
self.debug('kibanaLoaded = ' + kibanaLoaded);
|
||||
if (!kibanaLoaded) {
|
||||
var msg = 'Kibana is not loaded, retrying';
|
||||
self.debug(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
var loginPage = new RegExp('login').test(currentUrl);
|
||||
if (loginPage) {
|
||||
self.debug('Found loginPage = ' + loginPage + ', username = '
|
||||
+ config.servers.kibana.shield.username);
|
||||
return shieldPage.login(config.servers.kibana.shield.username,
|
||||
config.servers.kibana.shield.password)
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
});
|
||||
} else {
|
||||
return currentUrl;
|
||||
}
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
currentUrl = currentUrl.replace(/\/\/\w+:\w+@/, '//');
|
||||
var maxAdditionalLengthOnNavUrl = 230;
|
||||
// On several test failures at the end of the TileMap test we try to navigate back to
|
||||
// Visualize so we can create the next Vertical Bar Chart, but we can see from the
|
||||
// logging and the screenshot that it's still on the TileMap page. Why didn't the "get"
|
||||
// with a new timestamped URL go? I thought that sleep(700) between the get and the
|
||||
// refresh would solve the problem but didn't seem to always work.
|
||||
// So this hack fails the navSuccessful check if the currentUrl doesn't match the
|
||||
// appUrl plus up to 230 other chars.
|
||||
// Navigating to Settings when there is a default index pattern has a URL length of 196
|
||||
// (from debug output). Some other tabs may also be long. But a rather simple configured
|
||||
// visualization is about 1000 chars long. So at least we catch that case.
|
||||
var navSuccessful = new RegExp(appUrl + '.{0,' + maxAdditionalLengthOnNavUrl + '}$')
|
||||
.test(currentUrl);
|
||||
|
||||
if (!navSuccessful) {
|
||||
var msg = 'App failed to load: ' + appName +
|
||||
' in ' + defaultFindTimeout + 'ms' +
|
||||
' appUrl = ' + appUrl +
|
||||
' currentUrl = ' + currentUrl;
|
||||
self.debug(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
return currentUrl;
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
return doNavigation(appUrl)
|
||||
.then(function (currentUrl) {
|
||||
var lastUrl = currentUrl;
|
||||
return self.try(function () {
|
||||
// give the app time to update the URL
|
||||
return self.sleep(501)
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
self.debug('in doNavigation url = ' + currentUrl);
|
||||
if (lastUrl !== currentUrl) {
|
||||
lastUrl = currentUrl;
|
||||
throw new Error('URL changed, waiting for it to settle');
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
runScript: function (fn, timeout) {
|
||||
var self = this;
|
||||
// by default, give the app 10 seconds to load
|
||||
timeout = timeout || 10000;
|
||||
|
||||
// wait for deps on window before running script
|
||||
return self.remote
|
||||
.setExecuteAsyncTimeout(timeout)
|
||||
.executeAsync(function (done) {
|
||||
var interval = setInterval(function () {
|
||||
var ready = (document.readyState === 'complete');
|
||||
var hasJQuery = !!window.$;
|
||||
|
||||
if (ready && hasJQuery) {
|
||||
console.log('doc ready, jquery loaded');
|
||||
clearInterval(interval);
|
||||
done();
|
||||
}
|
||||
}, 10);
|
||||
}).then(function () {
|
||||
return self.remote.execute(fn);
|
||||
});
|
||||
},
|
||||
|
||||
getApp: function () {
|
||||
var self = this;
|
||||
|
||||
return self.remote.setFindTimeout(defaultFindTimeout)
|
||||
.findByCssSelector('.app-wrapper .application')
|
||||
.then(function () {
|
||||
return self.runScript(function () {
|
||||
var $ = window.$;
|
||||
var $scope = $('.app-wrapper .application').scope();
|
||||
return $scope ? $scope.chrome.getApp() : {};
|
||||
});
|
||||
});
|
||||
},
|
||||
|
||||
checkForKibanaApp: function () {
|
||||
var self = this;
|
||||
|
||||
return self.getApp()
|
||||
.then(function (app) {
|
||||
var appId = app.id;
|
||||
self.debug('current application: ' + appId);
|
||||
return appId === 'kibana';
|
||||
})
|
||||
.catch(function (err) {
|
||||
self.debug('kibana check failed');
|
||||
self.debug(err);
|
||||
// not on the kibana app...
|
||||
return false;
|
||||
});
|
||||
},
|
||||
|
||||
tryForTime: function (timeout, block) {
|
||||
var self = this;
|
||||
var start = Date.now();
|
||||
var retryDelay = 502;
|
||||
var lastTry = 0;
|
||||
var tempMessage;
|
||||
|
||||
function attempt() {
|
||||
lastTry = Date.now();
|
||||
|
||||
if (lastTry - start > timeout) {
|
||||
throw new Error('timeout ' + tempMessage);
|
||||
}
|
||||
|
||||
return Promise
|
||||
.try(block)
|
||||
.catch(function tryForTimeCatch(err) {
|
||||
self.debug('tryForTime failure: ' + err.message);
|
||||
tempMessage = err.message;
|
||||
return Promise.delay(retryDelay).then(attempt);
|
||||
});
|
||||
}
|
||||
|
||||
return Promise.try(attempt);
|
||||
},
|
||||
|
||||
try(block) {
|
||||
return this.tryForTime(defaultTryTimeout, block);
|
||||
},
|
||||
|
||||
log(...args) {
|
||||
console.log(moment().format('HH:mm:ss.SSS') + ':', util.format(...args));
|
||||
},
|
||||
|
||||
debug(...args) {
|
||||
if (config.debug) this.log(...args);
|
||||
},
|
||||
|
||||
sleep: function sleep(sleepMilliseconds) {
|
||||
var self = this;
|
||||
self.debug('... sleep(' + sleepMilliseconds + ') start');
|
||||
|
||||
return Promise.resolve().delay(sleepMilliseconds)
|
||||
.then(function () { self.debug('... sleep(' + sleepMilliseconds + ') end'); });
|
||||
},
|
||||
|
||||
handleError(testObj) {
|
||||
const testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
|
||||
return reason => {
|
||||
const now = Date.now();
|
||||
const fileName = `failure_${now}_${testName}`;
|
||||
|
||||
return this.saveScreenshot(fileName, true)
|
||||
function navigateTo(url) {
|
||||
return self.try(function () {
|
||||
// since we're using hash URLs, always reload first to force re-render
|
||||
self.debug('navigate to: ' + url);
|
||||
return self.remote.get(url)
|
||||
.then(function () {
|
||||
throw reason;
|
||||
return self.sleep(700);
|
||||
})
|
||||
.then(function () {
|
||||
self.debug('returned from get, calling refresh');
|
||||
return self.remote.refresh();
|
||||
})
|
||||
.then(function () {
|
||||
self.debug('check testStatusPage');
|
||||
if (testStatusPage !== false) {
|
||||
self.debug('self.checkForKibanaApp()');
|
||||
return self.checkForKibanaApp()
|
||||
.then(function (kibanaLoaded) {
|
||||
self.debug('kibanaLoaded = ' + kibanaLoaded);
|
||||
if (!kibanaLoaded) {
|
||||
var msg = 'Kibana is not loaded, retrying';
|
||||
self.debug(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
});
|
||||
}
|
||||
})
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
var loginPage = new RegExp('login').test(currentUrl);
|
||||
if (loginPage) {
|
||||
self.debug('Found loginPage = ' + loginPage + ', username = '
|
||||
+ config.servers.kibana.shield.username);
|
||||
return shieldPage.login(config.servers.kibana.shield.username,
|
||||
config.servers.kibana.shield.password)
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
});
|
||||
} else {
|
||||
return currentUrl;
|
||||
}
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
currentUrl = currentUrl.replace(/\/\/\w+:\w+@/, '//');
|
||||
var maxAdditionalLengthOnNavUrl = 230;
|
||||
// On several test failures at the end of the TileMap test we try to navigate back to
|
||||
// Visualize so we can create the next Vertical Bar Chart, but we can see from the
|
||||
// logging and the screenshot that it's still on the TileMap page. Why didn't the "get"
|
||||
// with a new timestamped URL go? I thought that sleep(700) between the get and the
|
||||
// refresh would solve the problem but didn't seem to always work.
|
||||
// So this hack fails the navSuccessful check if the currentUrl doesn't match the
|
||||
// appUrl plus up to 230 other chars.
|
||||
// Navigating to Settings when there is a default index pattern has a URL length of 196
|
||||
// (from debug output). Some other tabs may also be long. But a rather simple configured
|
||||
// visualization is about 1000 chars long. So at least we catch that case.
|
||||
var navSuccessful = new RegExp(appUrl + '.{0,' + maxAdditionalLengthOnNavUrl + '}$')
|
||||
.test(currentUrl);
|
||||
|
||||
if (!navSuccessful) {
|
||||
var msg = 'App failed to load: ' + appName +
|
||||
' in ' + defaultFindTimeout + 'ms' +
|
||||
' appUrl = ' + appUrl +
|
||||
' currentUrl = ' + currentUrl;
|
||||
self.debug(msg);
|
||||
throw new Error(msg);
|
||||
}
|
||||
|
||||
return currentUrl;
|
||||
});
|
||||
};
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
async saveScreenshot(fileName, isFailure = false) {
|
||||
try {
|
||||
const directoryName = isFailure ? 'failure' : 'session';
|
||||
const directoryPath = path.resolve(`test/screenshots/${directoryName}`);
|
||||
const filePath = path.resolve(directoryPath, `${fileName}.png`);
|
||||
this.debug(`Taking screenshot "${filePath}"`);
|
||||
return navigateTo(appUrl)
|
||||
.then(function (currentUrl) {
|
||||
var lastUrl = currentUrl;
|
||||
return self.try(function () {
|
||||
// give the app time to update the URL
|
||||
return self.sleep(501)
|
||||
.then(function () {
|
||||
return self.remote.getCurrentUrl();
|
||||
})
|
||||
.then(function (currentUrl) {
|
||||
self.debug('in navigateTo url = ' + currentUrl);
|
||||
if (lastUrl !== currentUrl) {
|
||||
lastUrl = currentUrl;
|
||||
throw new Error('URL changed, waiting for it to settle');
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const screenshot = await this.remote.takeScreenshot();
|
||||
await mkdirpAsync(directoryPath);
|
||||
await writeFileAsync(filePath, screenshot);
|
||||
} catch (err) {
|
||||
this.log(`SCREENSHOT FAILED: ${err}`);
|
||||
runScript(fn, timeout) {
|
||||
var self = this;
|
||||
// by default, give the app 10 seconds to load
|
||||
timeout = timeout || 10000;
|
||||
|
||||
// wait for deps on window before running script
|
||||
return self.remote
|
||||
.setExecuteAsyncTimeout(timeout)
|
||||
.executeAsync(function (done) {
|
||||
var interval = setInterval(function () {
|
||||
var ready = (document.readyState === 'complete');
|
||||
var hasJQuery = !!window.$;
|
||||
|
||||
if (ready && hasJQuery) {
|
||||
console.log('doc ready, jquery loaded');
|
||||
clearInterval(interval);
|
||||
done();
|
||||
}
|
||||
}, 10);
|
||||
}).then(function () {
|
||||
return self.remote.execute(fn);
|
||||
});
|
||||
}
|
||||
|
||||
getApp() {
|
||||
var self = this;
|
||||
|
||||
return self.remote.setFindTimeout(defaultFindTimeout)
|
||||
.findByCssSelector('.app-wrapper .application')
|
||||
.then(function () {
|
||||
return self.runScript(function () {
|
||||
var $ = window.$;
|
||||
var $scope = $('.app-wrapper .application').scope();
|
||||
return $scope ? $scope.chrome.getApp() : {};
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
checkForKibanaApp() {
|
||||
var self = this;
|
||||
|
||||
return self.getApp()
|
||||
.then(function (app) {
|
||||
var appId = app.id;
|
||||
self.debug('current application: ' + appId);
|
||||
return appId === 'kibana';
|
||||
})
|
||||
.catch(function (err) {
|
||||
self.debug('kibana check failed');
|
||||
self.debug(err);
|
||||
// not on the kibana app...
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
tryForTime(timeout, block) {
|
||||
var self = this;
|
||||
var start = Date.now();
|
||||
var retryDelay = 502;
|
||||
var lastTry = 0;
|
||||
var tempMessage;
|
||||
|
||||
function attempt() {
|
||||
lastTry = Date.now();
|
||||
|
||||
if (lastTry - start > timeout) {
|
||||
throw new Error('timeout ' + tempMessage);
|
||||
}
|
||||
},
|
||||
|
||||
findTestSubject: function findTestSubject(selector) {
|
||||
this.debug('in findTestSubject: ' + testSubjSelector(selector));
|
||||
return this.remote
|
||||
.setFindTimeout(defaultFindTimeout)
|
||||
.findDisplayedByCssSelector(testSubjSelector(selector));
|
||||
return bluebird
|
||||
.try(block)
|
||||
.catch(function tryForTimeCatch(err) {
|
||||
self.debug('tryForTime failure: ' + err.message);
|
||||
tempMessage = err.message;
|
||||
return bluebird.delay(retryDelay).then(attempt);
|
||||
});
|
||||
}
|
||||
|
||||
};
|
||||
return bluebird.try(attempt);
|
||||
}
|
||||
|
||||
return Common;
|
||||
}());
|
||||
try(block) {
|
||||
return this.tryForTime(defaultTryTimeout, block);
|
||||
}
|
||||
|
||||
log(...args) {
|
||||
console.log(moment().format('HH:mm:ss.SSS') + ':', util.format(...args));
|
||||
}
|
||||
|
||||
debug(...args) {
|
||||
if (config.debug) this.log(...args);
|
||||
}
|
||||
|
||||
sleep(sleepMilliseconds) {
|
||||
var self = this;
|
||||
self.debug('... sleep(' + sleepMilliseconds + ') start');
|
||||
|
||||
return bluebird.resolve().delay(sleepMilliseconds)
|
||||
.then(function () { self.debug('... sleep(' + sleepMilliseconds + ') end'); });
|
||||
}
|
||||
|
||||
handleError(testObj) {
|
||||
const testName = (testObj.parent) ? [testObj.parent.name, testObj.name].join('_') : testObj.name;
|
||||
return reason => {
|
||||
const now = Date.now();
|
||||
const fileName = `failure_${now}_${testName}`;
|
||||
|
||||
return this.saveScreenshot(fileName, true)
|
||||
.then(function () {
|
||||
throw reason;
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
async saveScreenshot(fileName, isFailure = false) {
|
||||
try {
|
||||
const directoryName = isFailure ? 'failure' : 'session';
|
||||
const directoryPath = path.resolve(`test/screenshots/${directoryName}`);
|
||||
const filePath = path.resolve(directoryPath, `${fileName}.png`);
|
||||
this.debug(`Taking screenshot "${filePath}"`);
|
||||
|
||||
const screenshot = await this.remote.takeScreenshot();
|
||||
await mkdirpAsync(directoryPath);
|
||||
await writeFileAsync(filePath, screenshot);
|
||||
} catch (err) {
|
||||
this.log(`SCREENSHOT FAILED: ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
findTestSubject(selector) {
|
||||
this.debug('in findTestSubject: ' + testSubjSelector(selector));
|
||||
return this.remote
|
||||
.setFindTimeout(defaultFindTimeout)
|
||||
.findDisplayedByCssSelector(testSubjSelector(selector));
|
||||
}
|
||||
|
||||
};
|
||||
|
|
14
test/visual_regression/home/_loading.js
Normal file
14
test/visual_regression/home/_loading.js
Normal file
|
@ -0,0 +1,14 @@
|
|||
import {
|
||||
bdd,
|
||||
scenarioManager,
|
||||
common,
|
||||
consolePage
|
||||
} from '../../support';
|
||||
|
||||
var expect = require('expect.js');
|
||||
|
||||
bdd.describe('Loading', function coverLoadingUi() {
|
||||
bdd.it('should show loading feebdack', async function () {
|
||||
// TODO: Take screenshots here.
|
||||
});
|
||||
});
|
16
test/visual_regression/home/index.js
Normal file
16
test/visual_regression/home/index.js
Normal file
|
@ -0,0 +1,16 @@
|
|||
import {
|
||||
bdd,
|
||||
remote,
|
||||
scenarioManager,
|
||||
defaultTimeout
|
||||
} from '../../support';
|
||||
|
||||
bdd.describe('Home', function () {
|
||||
this.timeout = defaultTimeout;
|
||||
|
||||
bdd.before(function () {
|
||||
return remote.setWindowSize(1200, 800);
|
||||
});
|
||||
|
||||
require('./_loading');
|
||||
});
|
26
test/visual_regression/index.js
Normal file
26
test/visual_regression/index.js
Normal file
|
@ -0,0 +1,26 @@
|
|||
define(function (require) {
|
||||
require('intern/dojo/node!../support/env_setup');
|
||||
|
||||
const bdd = require('intern!bdd');
|
||||
const intern = require('intern');
|
||||
const initCallbacks = [];
|
||||
|
||||
function onInit(callback) {
|
||||
initCallbacks.push(callback);
|
||||
}
|
||||
|
||||
global.__kibana__intern__ = { intern, bdd, onInit };
|
||||
|
||||
bdd.describe('Kibana visual regressions', function () {
|
||||
bdd.before(function () {
|
||||
initCallbacks.forEach(callback => {
|
||||
callback.call(this);
|
||||
});
|
||||
});
|
||||
|
||||
require([
|
||||
'intern/dojo/node!../support/index',
|
||||
'intern/dojo/node!./home',
|
||||
], function () {});
|
||||
});
|
||||
});
|
Loading…
Reference in a new issue