Re-enable Firefox functional testing on CI (#64159) (#64479)

* bring testing on Firefox back

* disable some tests

* skip more tests

* cut more suites for Firefox

* skip more tests for Firefox

* replace smoke tag with includeFirefox

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Dmitry Lemeshko 2020-04-25 17:43:12 +03:00 committed by GitHub
parent fd3f2b3dc4
commit d3f151fa2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
64 changed files with 39 additions and 74 deletions

4
Jenkinsfile vendored
View file

@ -11,7 +11,7 @@ kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
// 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
@ -28,7 +28,7 @@ kibanaPipeline(timeoutMinutes: 135, checkPrChanges: true) {
// 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
]),
'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
// 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),

View file

@ -153,16 +153,16 @@ A test suite is a collection of tests defined by calling `describe()`, and then
Use tags in `describe()` function to group functional tests. Tags include:
* `ciGroup{id}` - Assigns test suite to a specific CI worker
* `skipCloud` and `skipFirefox` - Excludes test suite from running on Cloud or Firefox
* `smoke` - Groups tests that run on Chrome and Firefox
* `includeFirefox` - Groups tests that run on Chrome and Firefox
**Cross-browser testing**:::
On CI, all the functional tests are executed in Chrome by default. To also run a suite against Firefox, assign the `smoke` tag:
On CI, all the functional tests are executed in Chrome by default. To also run a suite against Firefox, assign the `includeFirefox` tag:
["source","js"]
-----------
// on CI test suite will be run twice: in Chrome and Firefox
describe('My Cross-browser Test Suite', function () {
this.tags('smoke');
this.tags('includeFirefox');
it('My First Test');
}

View file

@ -40,7 +40,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'console']);
describe('console app', function describeIndexTests() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
log.debug('navigateTo console');
await PageObjects.common.navigateToApp('console');

View file

@ -33,7 +33,6 @@ export default function({ getService, getPageObjects }) {
// FLAKY: https://github.com/elastic/kibana/issues/53308
describe.skip('context link in discover', function contextSize() {
this.tags('smoke');
before(async function() {
await PageObjects.common.navigateToApp('discover');
await PageObjects.timePicker.setDefaultAbsoluteRange();

View file

@ -38,7 +38,7 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common', 'dashboard', 'header', 'visualize', 'timePicker']);
describe('dashboard filtering', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('dashboard/current/kibana');

View file

@ -24,7 +24,7 @@ export default function({ getPageObjects, getService }) {
const listingTable = getService('listingTable');
describe('dashboard save', function describeIndexTests() {
this.tags('smoke');
this.tags('includeFirefox');
const dashboardName = 'Dashboard Save Test';
const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key';

View file

@ -43,8 +43,6 @@ export default function({ getService, getPageObjects }) {
const dashboardName = 'Dashboard Panel Controls Test';
describe('dashboard panel controls', function viewEditModeTests() {
this.tags('smoke');
before(async function() {
await PageObjects.dashboard.initTests();
await PageObjects.dashboard.preserveCrossAppState();

View file

@ -27,7 +27,7 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['dashboard', 'timePicker', 'settings', 'common']);
describe('dashboard time zones', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('dashboard/current/kibana');

View file

@ -33,7 +33,6 @@ export default function({ getService, getPageObjects }) {
// FLAKY: https://github.com/elastic/kibana/issues/62281
describe.skip('doc link in discover', function contextSize() {
this.tags('smoke');
before(async function() {
await esArchiver.loadIfNeeded('logstash_functional');
await PageObjects.common.navigateToApp('discover');

View file

@ -28,7 +28,7 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']);
describe('discover tab', function describeIndexTests() {
this.tags('smoke');
this.tags('includeFirefox');
before(async function() {
await esArchiver.loadIfNeeded('logstash_functional');
await esArchiver.load('discover');

View file

@ -21,7 +21,7 @@ export default function({ getService, loadTestFile }) {
const browser = getService('browser');
describe('Getting Started ', function() {
this.tags(['ciGroup6', 'smoke']);
this.tags(['ciGroup6']);
before(async function() {
await browser.setWindowSize(1200, 800);

View file

@ -25,7 +25,7 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common', 'home']);
describe('Kibana takes you home', function describeIndexTests() {
this.tags('smoke');
this.tags('includeFirefox');
it('clicking on kibana logo should take you to home page', async () => {
await PageObjects.common.navigateToApp('settings');

View file

@ -32,8 +32,6 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard', 'timePicker']);
describe('sample data', function describeIndexTests() {
this.tags('smoke');
before(async () => {
await security.testUser.setRoles(['kibana_admin', 'kibana_sample_admin']);
await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');

View file

@ -25,8 +25,6 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['settings', 'common']);
describe('"Create Index Pattern" wizard', function() {
this.tags('smoke');
before(async function() {
// delete .kibana index and then wait for Kibana to re-create it
await kibanaServer.uiSettings.replace({});

View file

@ -24,8 +24,6 @@ export default ({ getService, getPageObjects }) => {
const PageObjects = getPageObjects(['visualize']);
describe('experimental visualizations in visualize app ', function() {
this.tags('smoke');
describe('experimental visualizations', () => {
beforeEach(async () => {
log.debug('navigateToApp visualize');

View file

@ -28,8 +28,6 @@ export default function({ getService, getPageObjects }) {
// FLAKY: https://github.com/elastic/kibana/issues/45089
describe('gauge chart', function indexPatternCreation() {
this.tags('smoke');
async function initGaugeVis() {
log.debug('navigateToApp visualize');
await PageObjects.visualize.navigateToNewVisualization();

View file

@ -25,7 +25,6 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['visualize', 'visEditor', 'visChart', 'timePicker']);
describe('heatmap chart', function indexPatternCreation() {
this.tags('smoke');
const vizName1 = 'Visualization HeatmapChart';
before(async function() {

View file

@ -24,7 +24,6 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['visualize', 'visEditor', 'visChart', 'timePicker']);
describe('inspector', function describeIndexTests() {
this.tags('smoke');
before(async function() {
await PageObjects.visualize.navigateToNewVisualization();
await PageObjects.visualize.clickVerticalBarChart();

View file

@ -29,7 +29,6 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
const PageObjects = getPageObjects(['visualize', 'visualBuilder', 'timePicker', 'visChart']);
describe('visual builder', function describeIndexTests() {
this.tags('smoke');
beforeEach(async () => {
await security.testUser.setRoles(['kibana_admin', 'test_logstash_reader']);
await PageObjects.visualize.navigateToNewVisualization();

View file

@ -27,7 +27,7 @@ export default function({ getService, getPageObjects }) {
const comboBox = getService('comboBox');
describe('chained controls', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await PageObjects.common.navigateToApp('visualize');

View file

@ -6,5 +6,5 @@ checks-reporter-with-killswitch "Firefox smoke test" \
node scripts/functional_tests \
--bail --debug \
--kibana-install-dir "$installDir" \
--include-tag "smoke" \
--include-tag "includeFirefox" \
--config test/functional/config.firefox.js;

View file

@ -6,5 +6,5 @@ checks-reporter-with-killswitch "X-Pack firefox smoke test" \
node scripts/functional_tests \
--debug --bail \
--kibana-install-dir "$KIBANA_INSTALL_DIR" \
--include-tag "smoke" \
--include-tag "includeFirefox" \
--config test/functional/config.firefox.js;

View file

@ -13,7 +13,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const security = getService('security');
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await security.testUser.setRoles(['kibana_admin']);
await pageObjects.common.navigateToApp('apiKeys');

View file

@ -15,7 +15,7 @@ export default function canvasSmokeTest({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['common']);
describe('smoke test', function() {
this.tags('smoke');
this.tags('includeFirefox');
const workpadListSelector = 'canvasWorkpadLoaderTable > canvasWorkpadLoaderWorkpad';
const testWorkpadId = 'workpad-1705f884-6224-47de-ba49-ca224fe6ec31';

View file

@ -12,7 +12,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const log = getService('log');
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await pageObjects.common.navigateToApp('crossClusterReplication');
});

View file

@ -12,7 +12,7 @@ export default function({ getService, getPageObjects }) {
const PageObjects = getPageObjects(['grokDebugger']);
describe('grok debugger app', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
// Increase window height to ensure "Simulate" button is shown above the

View file

@ -12,7 +12,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const log = getService('log');
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await pageObjects.common.navigateToApp('indexLifecycleManagement');
});

View file

@ -14,7 +14,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const browser = getService('browser');
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await pageObjects.common.navigateToApp('indexManagement');
});

View file

@ -15,7 +15,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const pageObjects = getPageObjects(['common', 'infraHome']);
describe('Home page', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('empty_kibana');
});

View file

@ -21,7 +21,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const traceId = '433b4651687e18be2c6c8e3b11f53d09';
describe('Infra link-to', function() {
this.tags('smoke');
it('redirects to the logs app and parses URL search params correctly', async () => {
const location = {
hash: '',

View file

@ -13,7 +13,7 @@ export default ({ getService }: FtrProviderContext) => {
const retry = getService('retry');
describe('Log Entry Categories Tab', function() {
this.tags('smoke');
this.tags('includeFirefox');
describe('with a trial license', () => {
it('is visible', async () => {

View file

@ -13,7 +13,7 @@ export default ({ getService }: FtrProviderContext) => {
const retry = getService('retry');
describe('Log Entry Rate Tab', function() {
this.tags('smoke');
this.tags('includeFirefox');
describe('with a trial license', () => {
it('is visible', async () => {

View file

@ -17,8 +17,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const retry = getService('retry');
describe('Logs Source Configuration', function() {
this.tags('smoke');
before(async () => {
await esArchiver.load('empty_kibana');
});

View file

@ -15,7 +15,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const pageObjects = getPageObjects(['common', 'infraHome']);
describe('Infrastructure Source Configuration', function() {
this.tags('smoke');
before(async () => {
await esArchiver.load('empty_kibana');
});

View file

@ -6,7 +6,7 @@
export default function({ loadTestFile }) {
describe('logstash', function() {
this.tags(['ciGroup2', 'smoke']);
this.tags(['ciGroup2']);
loadTestFile(require.resolve('./pipeline_list'));
loadTestFile(require.resolve('./pipeline_create'));

View file

@ -277,7 +277,7 @@ export default function({ getService }: FtrProviderContext) {
const calendarId = `wizard-test-calendar_${Date.now()}`;
describe('advanced job', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/ecommerce');
await ml.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');

View file

@ -57,7 +57,7 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('anomaly explorer', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -77,7 +77,7 @@ export default function({ getService }: FtrProviderContext) {
const calendarId = `wizard-test-calendar_${Date.now()}`;
describe('categorization', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/categorization');
await ml.testResources.createIndexPatternIfNeeded('ft_categorization', '@timestamp');

View file

@ -166,7 +166,7 @@ export default function({ getService }: FtrProviderContext) {
];
describe('job on data set with date_nanos time field', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/event_rate_nanos');
await ml.testResources.createIndexPatternIfNeeded(

View file

@ -74,7 +74,7 @@ export default function({ getService }: FtrProviderContext) {
const calendarId = `wizard-test-calendar_${Date.now()}`;
describe('multi metric', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -88,7 +88,7 @@ export default function({ getService }: FtrProviderContext) {
const calendarId = `wizard-test-calendar_${Date.now()}`;
describe('population', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/ecommerce');
await ml.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');

View file

@ -272,7 +272,7 @@ export default function({ getService }: FtrProviderContext) {
];
describe('saved search', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -73,7 +73,7 @@ export default function({ getService }: FtrProviderContext) {
const calendarId = `wizard-test-calendar_${Date.now()}`;
describe('single metric', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -40,7 +40,7 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('single metric viewer', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -12,7 +12,6 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('classification creation', function() {
this.tags(['smoke']);
before(async () => {
await esArchiver.loadIfNeeded('ml/bm_classification');
await ml.testResources.createIndexPatternIfNeeded('ft_bank_marketing', '@timestamp');

View file

@ -14,8 +14,6 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('jobs cloning supported by UI form', function() {
this.tags(['smoke']);
const testDataList: Array<{
suiteTitle: string;
archive: string;

View file

@ -12,7 +12,6 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('outlier detection creation', function() {
this.tags(['smoke']);
before(async () => {
await esArchiver.loadIfNeeded('ml/ihp_outlier');
await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier', '@timestamp');

View file

@ -12,7 +12,6 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('regression creation', function() {
this.tags(['smoke']);
before(async () => {
await esArchiver.loadIfNeeded('ml/egs_regression');
await ml.testResources.createIndexPatternIfNeeded('ft_egs_regression', '@timestamp');

View file

@ -34,7 +34,7 @@ export default function({ getService }: FtrProviderContext) {
];
describe('file based', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await ml.testResources.setKibanaTimeZoneToUTC();

View file

@ -376,7 +376,7 @@ export default function({ getService }: FtrProviderContext) {
}
describe('index based', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -10,7 +10,7 @@ export default function({ getService }: FtrProviderContext) {
const ml = getService('ml');
describe('page navigation', function() {
this.tags(['smoke', 'mlqa']);
this.tags(['includeFirefox', 'mlqa']);
before(async () => {
await ml.api.cleanMlIndices();
await ml.securityUI.loginAsMlPowerUser();

View file

@ -11,7 +11,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const pageObjects = getPageObjects(['common', 'remoteClusters']);
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await pageObjects.common.navigateToApp('remoteClusters');
});

View file

@ -16,7 +16,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) {
const spaces = getService('spaces');
describe('Security', function() {
this.tags('smoke');
this.tags('includeFirefox');
describe('Login Page', () => {
before(async () => {
await esArchiver.load('empty_kibana');

View file

@ -12,7 +12,6 @@ export default function({ getService, getPageObjects }) {
const esArchiver = getService('esArchiver');
describe('useremail', function() {
this.tags('smoke');
before(async () => {
await esArchiver.load('security/discover');
await PageObjects.settings.navigateTo();

View file

@ -12,7 +12,6 @@ export default function({ getService, getPageObjects }) {
const log = getService('log');
describe('users', function() {
this.tags('smoke');
before(async () => {
log.debug('users');
await PageObjects.settings.navigateTo();

View file

@ -13,7 +13,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
const es = getService('legacyEs');
describe('Home page', function() {
this.tags('smoke');
before(async () => {
await pageObjects.common.navigateToApp('snapshotRestore');
});

View file

@ -13,7 +13,7 @@ export default function enterSpaceFunctonalTests({
const PageObjects = getPageObjects(['security', 'spaceSelector']);
describe('Enter Space', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => {
await esArchiver.load('spaces/enter_space');
await PageObjects.security.forceLogout();

View file

@ -21,7 +21,7 @@ export default function spaceSelectorFunctonalTests({
]);
describe('Spaces', function() {
this.tags('smoke');
this.tags('includeFirefox');
describe('Space Selector', () => {
before(async () => {
await esArchiver.load('spaces/selector');

View file

@ -13,7 +13,7 @@ export default function statusPageFunctonalTests({
const PageObjects = getPageObjects(['security', 'statusPage', 'home']);
describe('Status Page', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => await esArchiver.load('empty_kibana'));
after(async () => await esArchiver.unload('empty_kibana'));

View file

@ -27,7 +27,6 @@ export default function({ getService }: FtrProviderContext) {
const transform = getService('transform');
describe('cloning', function() {
this.tags(['smoke']);
const transformConfig = getTransformConfig();
before(async () => {

View file

@ -18,7 +18,6 @@ export default function({ getService }: FtrProviderContext) {
const transform = getService('transform');
describe('creation_index_pattern', function() {
this.tags(['smoke']);
before(async () => {
await esArchiver.loadIfNeeded('ml/ecommerce');
await transform.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');

View file

@ -18,7 +18,6 @@ export default function({ getService }: FtrProviderContext) {
const transform = getService('transform');
describe('creation_saved_search', function() {
this.tags(['smoke']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await transform.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');

View file

@ -14,7 +14,7 @@ export default function upgradeAssistantFunctionalTests({
const PageObjects = getPageObjects(['upgradeAssistant']);
describe('Upgrade Checkup', function() {
this.tags('smoke');
this.tags('includeFirefox');
before(async () => await esArchiver.load('empty_kibana'));
after(async () => {
await PageObjects.upgradeAssistant.expectTelemetryHasFinish();

View file

@ -6,7 +6,7 @@
export default function({ loadTestFile }) {
describe('watcher app', function() {
this.tags(['ciGroup1', 'smoke']);
this.tags(['ciGroup1', 'includeFirefox']);
//loadTestFile(require.resolve('./management'));
loadTestFile(require.resolve('./watcher_test'));