From 581755098aa604dfb3c8123a0c2691ce28a84127 Mon Sep 17 00:00:00 2001 From: Dmitry Lemeshko Date: Mon, 17 Jun 2019 19:31:41 +0300 Subject: [PATCH] [firefox] Run functional tests in dedicated CI groups (#38416) * 2 jobs for Firefox tests * review fixes * make sh files executable * add check reporter --- .ci/jobs.yml | 2 ++ .ci/run.sh | 6 ++++++ test/functional/apps/console/_console.ts | 1 + .../apps/context/_discover_navigation.js | 1 + .../apps/dashboard/dashboard_filtering.js | 3 ++- .../apps/dashboard/dashboard_save.js | 1 + .../apps/dashboard/panel_controls.js | 2 ++ test/functional/apps/dashboard/time_zones.js | 3 ++- test/functional/apps/discover/_field_data.js | 1 + test/functional/apps/getting_started/index.js | 2 +- test/functional/apps/home/_home.js | 1 + test/functional/apps/home/_sample_data.js | 1 + .../_create_index_pattern_wizard.js | 1 + .../apps/visualize/_experimental_vis.js | 3 ++- .../functional/apps/visualize/_gauge_chart.js | 1 + .../apps/visualize/_heatmap_chart.js | 1 + test/functional/apps/visualize/_inspector.js | 1 + test/functional/apps/visualize/_tsvb_chart.ts | 1 + test/scripts/jenkins_ci_group.sh | 2 +- test/scripts/jenkins_firefox_smoke.sh | 19 +++++++++++++++++ test/scripts/jenkins_xpack_ci_group.sh | 1 - test/scripts/jenkins_xpack_firefox_smoke.sh | 21 +++++++++++++++++++ .../test/functional/apps/canvas/smoke_test.js | 9 ++++---- .../apps/grok_debugger/grok_debugger.js | 3 ++- .../test/functional/apps/infra/home_page.ts | 3 ++- .../functional/apps/rollup_job/rollup_jobs.js | 9 ++++---- .../functional/apps/security/user_email.js | 1 + x-pack/test/functional/apps/security/users.js | 1 + .../apps/status_page/status_page.ts | 3 ++- .../upgrade_assistant/upgrade_assistant.ts | 3 ++- x-pack/test/functional/apps/watcher/index.js | 2 +- 31 files changed, 88 insertions(+), 21 deletions(-) create mode 100755 test/scripts/jenkins_firefox_smoke.sh create mode 100755 test/scripts/jenkins_xpack_firefox_smoke.sh diff --git a/.ci/jobs.yml b/.ci/jobs.yml index d6249c28be7e..fc3e80064fa6 100644 --- a/.ci/jobs.yml +++ b/.ci/jobs.yml @@ -1,6 +1,7 @@ JOB: - kibana-intake - x-pack-intake + - kibana-firefoxSmoke - kibana-ciGroup1 - kibana-ciGroup2 - kibana-ciGroup3 @@ -16,6 +17,7 @@ JOB: # - kibana-visualRegression # make sure all x-pack-ciGroups are listed in test/scripts/jenkins_xpack_ci_group.sh + - x-pack-firefoxSmoke - x-pack-ciGroup1 - x-pack-ciGroup2 - x-pack-ciGroup3 diff --git a/.ci/run.sh b/.ci/run.sh index 21c77a0e345d..f0a18d929b51 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -20,6 +20,9 @@ kibana-ciGroup*) kibana-visualRegression*) ./test/scripts/jenkins_visual_regression.sh ;; +kibana-firefoxSmoke*) + ./test/scripts/jenkins_firefox_smoke.sh + ;; x-pack-intake) ./test/scripts/jenkins_xpack.sh ;; @@ -30,6 +33,9 @@ x-pack-ciGroup*) x-pack-visualRegression*) ./test/scripts/jenkins_xpack_visual_regression.sh ;; +x-pack-firefoxSmoke*) + ./test/scripts/jenkins_xpack_firefox_smoke.sh + ;; *) echo "JOB '$JOB' is not implemented." exit 1 diff --git a/test/functional/apps/console/_console.ts b/test/functional/apps/console/_console.ts index 92e6021eebf4..cdf2d3bf61f5 100644 --- a/test/functional/apps/console/_console.ts +++ b/test/functional/apps/console/_console.ts @@ -38,6 +38,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['common', 'console']); describe('console app', function describeIndexTests() { + this.tags('smoke'); before(async () => { log.debug('navigateTo console'); await PageObjects.common.navigateToApp('console'); diff --git a/test/functional/apps/context/_discover_navigation.js b/test/functional/apps/context/_discover_navigation.js index 19b43f5e3ffd..344d773214dc 100644 --- a/test/functional/apps/context/_discover_navigation.js +++ b/test/functional/apps/context/_discover_navigation.js @@ -31,6 +31,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'discover', 'timePicker']); describe('context link in discover', function contextSize() { + this.tags('smoke'); before(async function () { await PageObjects.common.navigateToApp('discover'); await PageObjects.timePicker.setAbsoluteRange(TEST_DISCOVER_START_TIME, TEST_DISCOVER_END_TIME); diff --git a/test/functional/apps/dashboard/dashboard_filtering.js b/test/functional/apps/dashboard/dashboard_filtering.js index c3f426e7d568..e586443e4909 100644 --- a/test/functional/apps/dashboard/dashboard_filtering.js +++ b/test/functional/apps/dashboard/dashboard_filtering.js @@ -34,7 +34,8 @@ export default function ({ getService, getPageObjects }) { const dashboardPanelActions = getService('dashboardPanelActions'); const PageObjects = getPageObjects(['dashboard', 'header', 'visualize']); - describe('dashboard filtering', async () => { + describe('dashboard filtering', async function () { + this.tags('smoke'); before(async () => { await PageObjects.dashboard.gotoDashboardLandingPage(); }); diff --git a/test/functional/apps/dashboard/dashboard_save.js b/test/functional/apps/dashboard/dashboard_save.js index 01e8e370e046..b4d2cec5722c 100644 --- a/test/functional/apps/dashboard/dashboard_save.js +++ b/test/functional/apps/dashboard/dashboard_save.js @@ -23,6 +23,7 @@ export default function ({ getPageObjects }) { const PageObjects = getPageObjects(['dashboard', 'header']); describe('dashboard save', function describeIndexTests() { + this.tags('smoke'); const dashboardName = 'Dashboard Save Test'; const dashboardNameEnterKey = 'Dashboard Save Test with Enter Key'; diff --git a/test/functional/apps/dashboard/panel_controls.js b/test/functional/apps/dashboard/panel_controls.js index 1fd8baaf4228..9c0c6c17906a 100644 --- a/test/functional/apps/dashboard/panel_controls.js +++ b/test/functional/apps/dashboard/panel_controls.js @@ -33,6 +33,8 @@ export default function ({ getService, getPageObjects }) { const dashboardName = 'Dashboard Panel Controls Test'; describe('dashboard panel controls', function viewEditModeTests() { + this.tags('smoke'); + before(async function () { await PageObjects.dashboard.initTests(); await browser.refresh(); diff --git a/test/functional/apps/dashboard/time_zones.js b/test/functional/apps/dashboard/time_zones.js index 10f91448c370..eca536a1389d 100644 --- a/test/functional/apps/dashboard/time_zones.js +++ b/test/functional/apps/dashboard/time_zones.js @@ -24,7 +24,8 @@ export default function ({ getService, getPageObjects }) { const pieChart = getService('pieChart'); const PageObjects = getPageObjects(['dashboard', 'timePicker', 'settings', 'common']); - describe('dashboard time zones', () => { + describe('dashboard time zones', function () { + this.tags('smoke'); before(async () => { await PageObjects.settings.navigateTo(); await PageObjects.settings.clickKibanaSavedObjects(); diff --git a/test/functional/apps/discover/_field_data.js b/test/functional/apps/discover/_field_data.js index f0bda7202232..828975445e1e 100644 --- a/test/functional/apps/discover/_field_data.js +++ b/test/functional/apps/discover/_field_data.js @@ -28,6 +28,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'header', 'discover', 'visualize', 'timePicker']); describe('discover tab', function describeIndexTests() { + this.tags('smoke'); before(async function () { const fromTime = '2015-09-19 06:31:44.000'; const toTime = '2015-09-23 18:31:44.000'; diff --git a/test/functional/apps/getting_started/index.js b/test/functional/apps/getting_started/index.js index c5a900be0d1b..e0495fb366e6 100644 --- a/test/functional/apps/getting_started/index.js +++ b/test/functional/apps/getting_started/index.js @@ -21,7 +21,7 @@ export default function ({ getService, loadTestFile }) { const browser = getService('browser'); describe('Getting Started ', function () { - this.tags('ciGroup6'); + this.tags(['ciGroup6', 'smoke']); before(async function () { await browser.setWindowSize(1200, 800); diff --git a/test/functional/apps/home/_home.js b/test/functional/apps/home/_home.js index f4a9b53d89ec..a889f84338ac 100644 --- a/test/functional/apps/home/_home.js +++ b/test/functional/apps/home/_home.js @@ -26,6 +26,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'home']); describe('Kibana takes you home', function describeIndexTests() { + this.tags('smoke'); it('clicking on kibana logo should take you to home page', async ()=> { await PageObjects.common.navigateToApp('settings'); diff --git a/test/functional/apps/home/_sample_data.js b/test/functional/apps/home/_sample_data.js index e77996224094..de14a0d1aaaf 100644 --- a/test/functional/apps/home/_sample_data.js +++ b/test/functional/apps/home/_sample_data.js @@ -27,6 +27,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard', 'timePicker']); describe('sample data', function describeIndexTests() { + this.tags('smoke'); before(async () => { await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData'); diff --git a/test/functional/apps/management/_create_index_pattern_wizard.js b/test/functional/apps/management/_create_index_pattern_wizard.js index 3539fd975b74..8a875551945b 100644 --- a/test/functional/apps/management/_create_index_pattern_wizard.js +++ b/test/functional/apps/management/_create_index_pattern_wizard.js @@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['settings', 'common']); describe('"Create Index Pattern" wizard', function () { + this.tags('smoke'); before(async function () { // delete .kibana index and then wait for Kibana to re-create it diff --git a/test/functional/apps/visualize/_experimental_vis.js b/test/functional/apps/visualize/_experimental_vis.js index 386185cd9fcc..8fdc9a18d990 100644 --- a/test/functional/apps/visualize/_experimental_vis.js +++ b/test/functional/apps/visualize/_experimental_vis.js @@ -23,7 +23,8 @@ export default ({ getService, getPageObjects }) => { const log = getService('log'); const PageObjects = getPageObjects(['common', 'visualize']); - describe('visualize app', () => { + describe('visualize app', function () { + this.tags('smoke'); describe('experimental visualizations', () => { diff --git a/test/functional/apps/visualize/_gauge_chart.js b/test/functional/apps/visualize/_gauge_chart.js index 55f0d4eaac61..3676ed0fd0b6 100644 --- a/test/functional/apps/visualize/_gauge_chart.js +++ b/test/functional/apps/visualize/_gauge_chart.js @@ -26,6 +26,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']); describe('gauge chart', function indexPatternCreation() { + this.tags('smoke'); const fromTime = '2015-09-19 06:31:44.000'; const toTime = '2015-09-23 18:31:44.000'; diff --git a/test/functional/apps/visualize/_heatmap_chart.js b/test/functional/apps/visualize/_heatmap_chart.js index 4395bddd77ab..90530c9a329d 100644 --- a/test/functional/apps/visualize/_heatmap_chart.js +++ b/test/functional/apps/visualize/_heatmap_chart.js @@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']); describe('heatmap chart', function indexPatternCreation() { + this.tags('smoke'); const vizName1 = 'Visualization HeatmapChart'; const fromTime = '2015-09-19 06:31:44.000'; const toTime = '2015-09-23 18:31:44.000'; diff --git a/test/functional/apps/visualize/_inspector.js b/test/functional/apps/visualize/_inspector.js index 2793bb640e11..47ecb5c7fcbe 100644 --- a/test/functional/apps/visualize/_inspector.js +++ b/test/functional/apps/visualize/_inspector.js @@ -25,6 +25,7 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['common', 'visualize', 'timePicker']); describe('inspector', function describeIndexTests() { + this.tags('smoke'); before(async function () { const fromTime = '2015-09-19 06:31:44.000'; const toTime = '2015-09-23 18:31:44.000'; diff --git a/test/functional/apps/visualize/_tsvb_chart.ts b/test/functional/apps/visualize/_tsvb_chart.ts index c34c5acf5ad4..0ca72d7fa764 100644 --- a/test/functional/apps/visualize/_tsvb_chart.ts +++ b/test/functional/apps/visualize/_tsvb_chart.ts @@ -30,6 +30,7 @@ export default function({ getService, getPageObjects }: FtrProviderContext) { const PageObjects = getPageObjects(['visualize', 'visualBuilder', 'timePicker']); describe('visual builder', function describeIndexTests() { + this.tags('smoke'); beforeEach(async () => { await PageObjects.visualize.navigateToNewVisualization(); await PageObjects.visualize.clickVisualBuilder(); diff --git a/test/scripts/jenkins_ci_group.sh b/test/scripts/jenkins_ci_group.sh index 80acdac7a985..fb532e9ec71d 100755 --- a/test/scripts/jenkins_ci_group.sh +++ b/test/scripts/jenkins_ci_group.sh @@ -18,4 +18,4 @@ if [ "$CI_GROUP" == "1" ]; then cd -; yarn run grunt run:pluginFunctionalTestsRelease --from=source; yarn run grunt run:interpreterFunctionalTestsRelease; -fi +fi \ No newline at end of file diff --git a/test/scripts/jenkins_firefox_smoke.sh b/test/scripts/jenkins_firefox_smoke.sh new file mode 100755 index 000000000000..bf3fe0691aa1 --- /dev/null +++ b/test/scripts/jenkins_firefox_smoke.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -e +trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT + +node scripts/build --debug --oss; +linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-oss-*-linux-x86_64.tar.gz')" +installDir="$PARENT_DIR/install/kibana" +mkdir -p "$installDir" +tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + +export TEST_BROWSER_HEADLESS=1 + +checks-reporter-with-killswitch "Firefox smoke test" \ + node scripts/functional_tests \ + --bail --debug \ + --kibana-install-dir "$installDir" \ + --include-tag "smoke" \ + --config test/functional/config.firefox.js; diff --git a/test/scripts/jenkins_xpack_ci_group.sh b/test/scripts/jenkins_xpack_ci_group.sh index 9bdcd7286292..3527aa5eedfa 100755 --- a/test/scripts/jenkins_xpack_ci_group.sh +++ b/test/scripts/jenkins_xpack_ci_group.sh @@ -46,4 +46,3 @@ echo "" # --config "test/functional/config.firefox.js" # echo "" # echo "" - diff --git a/test/scripts/jenkins_xpack_firefox_smoke.sh b/test/scripts/jenkins_xpack_firefox_smoke.sh new file mode 100755 index 000000000000..783e37fdfb51 --- /dev/null +++ b/test/scripts/jenkins_xpack_firefox_smoke.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e +trap 'node "$KIBANA_DIR/src/dev/failed_tests/cli"' EXIT + +node scripts/build --debug --no-oss; +linuxBuild="$(find "$KIBANA_DIR/target" -name 'kibana-*-linux-x86_64.tar.gz')" +installDir="$PARENT_DIR/install/kibana" +mkdir -p "$installDir" +tar -xzf "$linuxBuild" -C "$installDir" --strip=1 + +export TEST_BROWSER_HEADLESS=1 + +cd "$XPACK_DIR" + +checks-reporter-with-killswitch "X-Pack firefox smoke test" \ + node scripts/functional_tests \ + --debug --bail \ + --kibana-install-dir "$installDir" \ + --include-tag "smoke" \ + --config test/functional/config.firefox.js; diff --git a/x-pack/test/functional/apps/canvas/smoke_test.js b/x-pack/test/functional/apps/canvas/smoke_test.js index b309472f382c..744cebfdc263 100644 --- a/x-pack/test/functional/apps/canvas/smoke_test.js +++ b/x-pack/test/functional/apps/canvas/smoke_test.js @@ -14,16 +14,15 @@ export default function canvasSmokeTest({ getService, getPageObjects }) { const retry = getService('retry'); const PageObjects = getPageObjects(['common']); - describe('smoke test', async () => { + describe('smoke test', async function () { + this.tags('smoke'); const workpadListSelector = 'canvasWorkpadLoaderTable canvasWorkpadLoaderWorkpad'; const testWorkpadId = 'workpad-1705f884-6224-47de-ba49-ca224fe6ec31'; before(async () => { // init data - await Promise.all([ - esArchiver.loadIfNeeded('logstash_functional'), - esArchiver.load('canvas/default'), - ]); + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.load('canvas/default'); // load canvas // see also navigateToUrl(app, hash) diff --git a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js index bd77d93a94dc..a8092f5acb84 100644 --- a/x-pack/test/functional/apps/grok_debugger/grok_debugger.js +++ b/x-pack/test/functional/apps/grok_debugger/grok_debugger.js @@ -11,7 +11,8 @@ export default function ({ getService, getPageObjects }) { const PageObjects = getPageObjects(['grokDebugger']); - describe('grok debugger app', () => { + describe('grok debugger app', function () { + this.tags('smoke'); before(async () => { await esArchiver.load('empty_kibana'); // Increase window height to ensure "Simulate" button is shown above the diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts index fe18d33723ab..bdd08d170650 100644 --- a/x-pack/test/functional/apps/infra/home_page.ts +++ b/x-pack/test/functional/apps/infra/home_page.ts @@ -15,7 +15,8 @@ export default ({ getPageObjects, getService }: KibanaFunctionalTestDefaultProvi const esArchiver = getService('esArchiver'); const pageObjects = getPageObjects(['common', 'infraHome']); - describe('Home page', () => { + describe('Home page', function() { + this.tags('smoke'); before(async () => { await esArchiver.load('empty_kibana'); }); diff --git a/x-pack/test/functional/apps/rollup_job/rollup_jobs.js b/x-pack/test/functional/apps/rollup_job/rollup_jobs.js index 5546cd9311ef..0d8782087466 100644 --- a/x-pack/test/functional/apps/rollup_job/rollup_jobs.js +++ b/x-pack/test/functional/apps/rollup_job/rollup_jobs.js @@ -12,13 +12,12 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); const PageObjects = getPageObjects(['security', 'rollup', 'common', 'header']); - describe('rollup job', async () => { + describe('rollup job', async function () { + this.tags('smoke'); before(async () => { // init data - await Promise.all([ - esArchiver.loadIfNeeded('logstash_functional'), - esArchiver.load('canvas/default'), - ]); + await esArchiver.loadIfNeeded('logstash_functional'); + await esArchiver.load('canvas/default'); await PageObjects.common.navigateToApp('rollupJob'); }); diff --git a/x-pack/test/functional/apps/security/user_email.js b/x-pack/test/functional/apps/security/user_email.js index b79e32fc6a7d..538066d19ff1 100644 --- a/x-pack/test/functional/apps/security/user_email.js +++ b/x-pack/test/functional/apps/security/user_email.js @@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }) { const esArchiver = getService('esArchiver'); describe('useremail', function () { + this.tags('smoke'); before(async () => { await esArchiver.load('security/discover'); await PageObjects.settings.navigateTo(); diff --git a/x-pack/test/functional/apps/security/users.js b/x-pack/test/functional/apps/security/users.js index 0408b0723d56..459fb3e495c5 100644 --- a/x-pack/test/functional/apps/security/users.js +++ b/x-pack/test/functional/apps/security/users.js @@ -13,6 +13,7 @@ export default function ({ getService, getPageObjects }) { const log = getService('log'); describe('users', function () { + this.tags('smoke'); before(async () => { log.debug('users'); diff --git a/x-pack/test/functional/apps/status_page/status_page.ts b/x-pack/test/functional/apps/status_page/status_page.ts index 88203b350c26..6df02589a9ce 100644 --- a/x-pack/test/functional/apps/status_page/status_page.ts +++ b/x-pack/test/functional/apps/status_page/status_page.ts @@ -10,7 +10,8 @@ export default function statusPageFunctonalTests({ getService, getPageObjects }: const esArchiver = getService('esArchiver'); const PageObjects = getPageObjects(['security', 'statusPage', 'home']); - describe('Status Page', () => { + describe('Status Page', function() { + this.tags('smoke'); before(async () => await esArchiver.load('empty_kibana')); after(async () => await esArchiver.unload('empty_kibana')); diff --git a/x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts b/x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts index 788146a7f2a1..a3b340d99cdf 100644 --- a/x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts +++ b/x-pack/test/functional/apps/upgrade_assistant/upgrade_assistant.ts @@ -14,7 +14,8 @@ export default function upgradeAssistantFunctionalTests({ const esArchiver = getService('esArchiver'); const PageObjects = getPageObjects(['upgradeAssistant']); - describe('Upgrade Checkup', () => { + describe('Upgrade Checkup', function() { + this.tags('smoke'); before(async () => await esArchiver.load('empty_kibana')); after(async () => { await PageObjects.upgradeAssistant.expectTelemetryHasFinish(); diff --git a/x-pack/test/functional/apps/watcher/index.js b/x-pack/test/functional/apps/watcher/index.js index 5542f5cb8f0b..6baab838f283 100644 --- a/x-pack/test/functional/apps/watcher/index.js +++ b/x-pack/test/functional/apps/watcher/index.js @@ -6,7 +6,7 @@ export default function ({ loadTestFile }) { describe('watcher app', function () { - this.tags('ciGroup1'); + this.tags(['ciGroup1', 'smoke']); //loadTestFile(require.resolve('./management')); loadTestFile(require.resolve('./watcher_test'));