Automated package testing (#88900) (#100812)

Co-authored-by: Tyler Smalley <tylersmalley@me.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Jonathan Budzenski <jon@budzenski.me>
Co-authored-by: Tyler Smalley <tylersmalley@me.com>

.ci/package-testing/Jenkinsfile

@@ -0,0 +1,29 @@
+#!/bin/groovy
+library 'kibana-pipeline-library'
+kibanaLibrary.load()
+kibanaPipeline(timeoutMinutes: 300) {
+  slackNotifications.onFailure {
+    ciStats.trackBuild {
+      workers.ci(ramDisk: false, name: "package-build", size: 'l', runErrorReporter: false) {
+        withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') {
+          kibanaPipeline.bash("test/scripts/jenkins_xpack_package_build.sh", "Package builds")
+        }
+      }
+      def packageTypes = ['deb', 'docker', 'rpm']
+      def workers = [:]
+      packageTypes.each { type ->
+        workers["package-${type}"] = {
+          testPackage(type)
+        }
+      }
+      parallel(workers)
+    }
+  }
+}
+def testPackage(packageType) {
+  workers.ci(ramDisk: false, name: "package-${packageType}", size: 's', runErrorReporter: false) {
+    withGcpServiceAccount.fromVaultSecret('secret/kibana-issues/dev/ci-artifacts-key', 'value') {
+      kibanaPipeline.bash("test/scripts/jenkins_xpack_package_${packageType}.sh", "Execute package testing for ${packageType}")
+    }
+  }
+}

.gitignore

@@ -60,6 +60,7 @@ npm-debug.log*
 .ci/runbld
 .ci/bash_standard_lib.sh
 .gradle
+.vagrant
 
 ## @cypress/snapshot from apm plugin
 snapshots.js

docs/developer/contributing/development-package-tests.asciidoc

@@ -0,0 +1,64 @@
+[[development-package-tests]]
+== Package Testing
+
+Packaging tests use Vagrant virtual machines as hosts and Ansible for
+provisioning and assertions. Kibana distributions are copied from the
+target folder into each VM and installed, along with required
+dependencies.
+
+=== Setup
+
+* https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html[Ansible]
++
+```
+# Ubuntu
+sudo apt-get install python3-pip libarchive-tools
+pip3 install --user ansible
+
+# Darwin
+brew install python3
+pip3 install --user ansible
+```
+* https://www.vagrantup.com/downloads[Vagrant]
+* https://www.virtualbox.org/wiki/Downloads[Virtualbox]
+
+=== Machines
+
+[cols=",,",options="header",]
+|===
+|Hostname |IP |Description
+|deb |192.168.50.5 |Installation of Kibana’s deb package
+|rpm |192.168.50.6 |Installation of Kibana’s rpm package
+|docker |192.168.50.7 |Installation of Kibana’s docker image
+|===
+
+=== Running
+
+```
+# Build distributions
+node scripts/build --all-platforms --debug --no-oss
+
+cd test/package
+
+# Setup virtual machine and networking
+vagrant up <hostname> --no-provision
+
+# Install Kibana and run OS level tests
+# This step can be repeated when adding new tests, it ensures machine state - installations won't run twice
+vagrant provision <hostname>
+
+# Running functional tests
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.50.1 \
+  -E discovery.type=single-node \
+  --license=trial
+TEST_KIBANA_URL=http://elastic:changeme@<ip>:5601 \
+TEST_ES_URL=http://elastic:changeme@192.168.50.1:9200 \
+  node scripts/functional_test_runner.js --include-tag=smoke
+```
+
+=== Cleanup
+
+....
+vagrant destroy <hostname>
+....

docs/developer/contributing/development-tests.asciidoc

@@ -74,6 +74,7 @@ to learn more about using the node scripts we provide for building
 * <<development-functional-tests>>
 * <<development-unit-tests>>
 * <<development-accessibility-tests>>
+* <<development-package-tests>>
 
 include::development-functional-tests.asciidoc[leveloffset=+1]
 
@@ -81,6 +82,8 @@ include::development-unit-tests.asciidoc[leveloffset=+1]
 
 include::development-accessibility-tests.asciidoc[leveloffset=+1]
 
+include::development-package-tests.asciidoc[leveloffset=+1]
+
 [discrete]
 === Cross-browser compatibility
 

src/dev/precommit_hook/casing_check_config.js

@@ -39,6 +39,7 @@ export const IGNORE_FILE_GLOBS = [
   'vars/*',
   '.ci/pipeline-library/**/*',
   'packages/kbn-test/jest-preset.js',
+  'test/package/Vagrantfile',
 
   // filename must match language code which requires capital letters
   '**/translations/*.json',

test/package/Vagrantfile

@@ -0,0 +1,27 @@
+Vagrant.configure("2") do |config|
+  config.vm.synced_folder '../../target/', '/packages'
+
+  config.vm.define "deb" do |deb|
+    deb.vm.box = 'elastic/debian-9-x86_64'
+    deb.vm.provision "ansible" do |ansible|
+      ansible.playbook = "deb.yml"
+    end
+    deb.vm.network "private_network", ip: "192.168.50.5"
+  end
+
+  config.vm.define "rpm" do |rpm|
+    rpm.vm.box = 'elastic/centos-7-x86_64'
+    rpm.vm.provision "ansible" do |ansible|
+      ansible.playbook = "rpm.yml"
+    end
+    rpm.vm.network "private_network", ip: "192.168.50.6"
+  end
+
+  config.vm.define "docker" do |docker|
+    docker.vm.box = 'elastic/ubuntu-18.04-x86_64'
+    docker.vm.provision "ansible" do |ansible|
+      ansible.playbook = "docker.yml"
+    end
+    docker.vm.network "private_network", ip: "192.168.50.7"
+  end
+end

test/package/deb.yml

@@ -0,0 +1,11 @@
+- name: test kibana deb package
+  hosts: deb
+  roles:
+    - install_kibana_deb
+    - assert_keystore_available
+    - assert_keystore_cli
+    - assert_kibana_yml
+    - assert_kibana_listening
+    - assert_kibana_available
+    - assert_kibana_log
+    - assert_kibana_data

test/package/docker.yml

@@ -0,0 +1,7 @@
+- name: test kibana docker package
+  hosts: docker
+  roles:
+    - install_docker
+    - install_kibana_docker
+    - assert_kibana_listening
+    - assert_kibana_available

test/package/group_vars/all.yml

@@ -0,0 +1,2 @@
+elasticsearch_username: kibana_system
+elasticsearch_password: changeme

test/package/roles/assert_keystore_available/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: stat
+  become: yes
+  register: keystore
+  stat:
+    path: /etc/kibana/kibana.keystore
+
+- name: 0660 root:kibana
+  assert:
+    that:
+      - keystore.stat.exists
+      - keystore.stat.mode == "0660"
+      - keystore.stat.pw_name == "root"
+      - keystore.stat.gr_name == "kibana"

test/package/roles/assert_keystore_cli/tasks/main.yml

@@ -0,0 +1,22 @@
+- name: "add server.name: package-testing"
+  become: yes
+  command: 
+    cmd: /usr/share/kibana/bin/kibana-keystore add server.name --stdin
+    stdin: package-testing
+  register: kibana_keystore_add
+
+- debug:
+    msg: "{{ kibana_keystore_add.stdout }}"
+
+- name: register kibana-keystore list
+  become: yes
+  command: /usr/share/kibana/bin/kibana-keystore list 
+  register: kibana_keystore_list
+
+- debug: 
+    msg: "{{ kibana_keystore_list.stdout }}"
+
+- name: assert kibana-keystore list contains server.name
+  assert:
+    that:
+      - kibana_keystore_list.stdout == "server.name"

test/package/roles/assert_kibana_available/tasks/main.yml

@@ -0,0 +1,8 @@
+- name: "localhost:5601/api/status"
+  uri:
+    url: "http://localhost:5601/api/status"
+    status_code: [200, 401]
+  register: result
+  until: result.status != 503
+  retries: 3
+  delay: 30

test/package/roles/assert_kibana_data/tasks/main.yml

@@ -0,0 +1,13 @@
+- name: stat /var/lib/kibana
+  become: yes
+  register: kibana_data_directory
+  stat:
+    path: /var/lib/kibana
+
+- name: /var/lib/kibana 2750 kibana:kibana
+  assert:
+    that:
+      - kibana_log_directory.stat.exists
+      - kibana_log_directory.stat.mode == "2750"
+      - kibana_log_directory.stat.pw_name == "kibana"
+      - kibana_log_directory.stat.gr_name == "kibana"

test/package/roles/assert_kibana_listening/tasks/main.yml

@@ -0,0 +1,2 @@
+- name: localhost:5601
+  wait_for: host=localhost port=5601 timeout=60

test/package/roles/assert_kibana_log/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: stat /var/log/kibana
+  become: yes
+  register: kibana_log_directory
+  stat:
+    path: /var/log/kibana
+
+- name: /var/log/kibana 2750 kibana:kibana
+  assert:
+    that:
+      - kibana_log_directory.stat.exists
+      - kibana_log_directory.stat.mode == "2750"
+      - kibana_log_directory.stat.pw_name == "kibana"
+      - kibana_log_directory.stat.gr_name == "kibana"
+
+- name: stat /var/log/kibana/kibana.log
+  become: yes
+  register: kibana_log
+  stat:
+    path: /var/log/kibana/kibana.log
+
+- name: /var/log/kibana/kibana.log 0644 kibana:kibana
+  assert:
+    that:
+      - kibana_log.stat.exists
+      - kibana_log.stat.mode == "0644"
+      - kibana_log.stat.pw_name == "kibana"
+      - kibana_log.stat.gr_name == "kibana"

test/package/roles/assert_kibana_pid/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: stat /run/kibana
+  become: yes
+  register: kibana_pid_directory
+  stat:
+    path: /run/kibana
+
+- name: /run/kibana 0775 kibana:kibana
+  assert:
+    that:
+      - kibana_pid_directory.stat.exists
+      - kibana_pid_directory.stat.mode == "0775"
+      - kibana_pid_directory.stat.pw_name == "kibana"
+      - kibana_pid_directory.stat.gr_name == "kibana"
+
+- name: stat /run/kibana/kibana.pid
+  become: yes
+  register: kibana_pid
+  stat:
+    path: /run/kibana/kibana.pid
+
+- name: /run/kibana/kibana.pid 0644 kibana:kibana
+  assert:
+    that:
+      - kibana_pid.stat.exists
+      - kibana_pid.stat.mode == "0644"
+      - kibana_pid.stat.pw_name == "kibana"
+      - kibana_pid.stat.gr_name == "kibana"

test/package/roles/assert_kibana_yml/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: stat /etc/kibana
+  become: yes
+  register: kibana_yml_directory
+  stat:
+    path: /etc/kibana
+
+- name: /etc/kibana 2750 root:kibana
+  assert:
+    that:
+      - kibana_yml_directory.stat.exists
+      - kibana_yml_directory.stat.mode == "2750"
+      - kibana_yml_directory.stat.pw_name == "root"
+      - kibana_yml_directory.stat.gr_name == "kibana"
+
+- name: stat /etc/kibana/kibana.yml
+  become: yes
+  register: kibana_yml
+  stat:
+    path: /etc/kibana/kibana.yml
+
+- name: /etc/kibana/kibana.yml 0660 root:kibana
+  assert:
+    that:
+      - kibana_yml.stat.exists
+      - kibana_yml.stat.mode == "0660"
+      - kibana_yml.stat.pw_name == "root"
+      - kibana_yml.stat.gr_name == "kibana"

test/package/roles/install_docker/tasks/main.yml

@@ -0,0 +1,39 @@
+- name: install dependencies
+  become: yes
+  apt:
+    name: '{{ docker_dependencies }}'
+    state: present
+    update_cache: yes
+
+- name: add docker gpg key
+  become: yes
+  apt_key:
+    url: https://download.docker.com/linux/ubuntu/gpg
+    state: present
+
+- name: fetch ubuntu version
+  shell: lsb_release -cs
+  register: ubuntu_version
+  changed_when: false
+
+- name: add docker repository
+  become: yes
+  apt_repository:
+    repo: 'deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ubuntu_version.stdout }} stable'
+    state: present
+
+- name: update apt packages
+  become: yes
+  apt:
+    update_cache: yes
+
+- name: install docker
+  become: yes
+  apt:
+    name: 'docker-ce'
+    state: present
+
+- name: install docker sdk
+  become: yes
+  pip:
+    name: docker

test/package/roles/install_docker/vars/main.yml

@@ -0,0 +1,7 @@
+docker_dependencies:
+  - apt-transport-https
+  - ca-certificates
+  - curl
+  - gnupg-agent
+  - software-properties-common
+  - python3-pip

test/package/roles/install_kibana_deb/tasks/main.yml

@@ -0,0 +1,34 @@
+- name: install dependencies
+  become: yes
+  apt:
+    name:
+      - libnss3
+      - fonts-liberation
+      - libfontconfig
+    state: latest
+
+- name: find deb package
+  find:
+    paths: /packages/
+    patterns: kibana-*-amd64.deb
+  register: kibana_deb
+
+- name: install
+  become: yes
+  apt:
+    deb: "{{ kibana_deb.files[0].path }}"
+    state: present
+
+- name: copy configuration
+  become: yes
+  template:
+    src: templates/kibana.yml
+    dest: /etc/kibana/kibana.yml
+  register: config
+
+- name: start kibana
+  become: yes
+  systemd:
+    state: started
+    name: kibana
+    daemon_reload: yes

test/package/roles/install_kibana_docker/tasks/main.yml

@@ -0,0 +1,26 @@
+- name: find docker image
+  find:
+    paths: /packages/
+    patterns: kibana-*-docker-image.tar.gz
+  register: kibana_docker
+
+- name: load image
+  become: yes
+  docker_image:
+    name: kibana
+    load_path: "{{ kibana_docker.files[0].path }}"
+    timeout: 300
+    source: load
+    state: present
+
+- name: start kibana
+  become: yes
+  docker_container:
+    name: kibana
+    image: "{{ kibana_docker.files[0].path | basename| regex_replace('kibana-(.*)-docker-image.tar.gz', 'docker.elastic.co/kibana/kibana:\\1') }}"
+    network_mode: host
+    env:
+      SERVER_HOST: 0.0.0.0
+      ELASTICSEARCH_HOSTS: http://192.168.50.1:9200
+      ELASTICSEARCH_USERNAME: "{{ elasticsearch_username }}"
+      ELASTICSEARCH_PASSWORD: "{{ elasticsearch_password }}"

test/package/roles/install_kibana_rpm/tasks/main.yml

@@ -0,0 +1,45 @@
+- name: install dependencies
+  become: yes
+  yum:
+    name:
+      - nss
+      - fontconfig
+      - freetype
+    state: latest
+
+- name: find rpm package
+  find:
+    paths: /packages/
+    patterns: kibana-*-x86_64.rpm
+  register: kibana_rpm
+
+- name: install
+  become: yes
+  yum:
+    name: "{{ kibana_rpm.files[0].path }}"
+    state: present
+    disable_gpg_check: yes
+
+- name: copy configuration
+  become: yes
+  template:
+    src: templates/kibana.yml
+    dest: /etc/kibana/kibana.yml
+  register: config
+
+- name: open port 5601/tcp
+  become: yes
+  command:
+    cmd: firewall-cmd --zone=public --permanent --add-port=5601/tcp
+
+- name: reload firewall
+  become: yes
+  command:
+    cmd: firewall-cmd --reload
+
+- name: start kibana
+  become: yes
+  systemd:
+    state: started
+    name: kibana
+    daemon_reload: yes

test/package/roles/upgrade_apt_packages/tasks/main.yml

@@ -0,0 +1,6 @@
+- name: upgrade apt packages
+  become: yes
+  apt:
+    name: '*'
+    state: latest
+    update_cache: yes

test/package/roles/upgrade_yum_packages/tasks/main.yml

@@ -0,0 +1,6 @@
+- name: upgrade yum packages
+  become: yes
+  yum:
+    name: '*'
+    state: latest
+    update_cache: yes

test/package/rpm.yml

@@ -0,0 +1,11 @@
+- name: test kibana rpm package
+  hosts: rpm
+  roles:
+    - install_kibana_rpm
+    - assert_keystore_available
+    - assert_keystore_cli
+    - assert_kibana_yml
+    - assert_kibana_listening
+    - assert_kibana_available
+    - assert_kibana_log
+    - assert_kibana_data

test/package/templates/kibana.yml

@@ -0,0 +1,5 @@
+server.host: 0.0.0.0
+
+elasticsearch.hosts: http://192.168.50.1:9200
+elasticsearch.username: "{{ elasticsearch_username }}"
+elasticsearch.password: "{{ elasticsearch_password }}"

test/scripts/jenkins_xpack_package_build.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+source src/dev/ci_setup/setup_env.sh
+
+export TMP=/tmp
+export TMPDIR=/tmp
+
+node scripts/build --all-platforms --debug --no-oss
+
+gsutil -q -m cp 'target/*' "gs://ci-artifacts.kibana.dev/package-testing/$GIT_COMMIT/"

test/scripts/jenkins_xpack_package_deb.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+source src/dev/ci_setup/setup_env.sh
+
+gsutil -q -m cp "gs://ci-artifacts.kibana.dev/package-testing/$GIT_COMMIT/kibana-*.deb" ./target
+
+export VAGRANT_CWD=test/package
+vagrant up deb --no-provision
+
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.50.1 \
+  -E discovery.type=single-node \
+  --license=trial &
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+
+vagrant provision deb
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL=http://elastic:changeme@192.168.50.5:5601
+export TEST_ES_URL=http://elastic:changeme@192.168.50.1:9200
+
+cd x-pack
+node scripts/functional_test_runner.js --include-tag=smoke

test/scripts/jenkins_xpack_package_docker.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+source src/dev/ci_setup/setup_env.sh
+
+gsutil -q -m cp "gs://ci-artifacts.kibana.dev/package-testing/$GIT_COMMIT/kibana-[0-9]*-docker-image.tar.gz" ./target
+
+export VAGRANT_CWD=test/package
+vagrant up docker --no-provision
+
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.50.1 \
+  -E discovery.type=single-node \
+  --license=trial &
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+
+vagrant provision docker
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL=http://elastic:changeme@192.168.50.7:5601
+export TEST_ES_URL=http://elastic:changeme@192.168.50.1:9200
+
+cd x-pack
+node scripts/functional_test_runner.js --include-tag=smoke

test/scripts/jenkins_xpack_package_rpm.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -e
+
+source src/dev/ci_setup/setup_env.sh
+
+gsutil -q -m cp "gs://ci-artifacts.kibana.dev/package-testing/$GIT_COMMIT/kibana-*.rpm" ./target
+
+export VAGRANT_CWD=test/package
+vagrant up rpm --no-provision
+
+node scripts/es snapshot \
+  -E network.bind_host=127.0.0.1,192.168.50.1 \
+  -E discovery.type=single-node \
+  --license=trial &
+while ! timeout 1 bash -c "echo > /dev/tcp/localhost/9200"; do sleep 30; done
+
+vagrant provision rpm
+
+export TEST_BROWSER_HEADLESS=1
+export TEST_KIBANA_URL=http://elastic:changeme@192.168.50.6:5601
+export TEST_ES_URL=http://elastic:changeme@192.168.50.1:9200
+
+cd x-pack
+node scripts/functional_test_runner.js --include-tag=smoke

vars/workers.groovy

@@ -103,6 +103,7 @@ def base(Map params, Closure closure) {
       "PR_TARGET_BRANCH=${env.ghprbTargetBranch ?: ''}",
       "PR_AUTHOR=${env.ghprbPullAuthorLogin ?: ''}",
       "TEST_BROWSER_HEADLESS=1",
+      "GIT_COMMIT=${checkoutInfo.commit}",
       "GIT_BRANCH=${checkoutInfo.branch}",
       "TMPDIR=${env.WORKSPACE}/tmp", // For Chrome and anything else that respects it
       "BUILD_TS_REFS_DISABLE=true", // no need to build ts refs in bootstrap

x-pack/test/functional/apps/reporting/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('Reporting', function () {
+    loadTestFile(require.resolve('./reporting'));
+  });
+}

x-pack/test/functional/apps/reporting/reporting.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../ftr_provider_context';
+
+export default function ({ getService, getPageObjects }: FtrProviderContext) {
+  const pageObjects = getPageObjects(['dashboard', 'common', 'reporting']);
+  const es = getService('es');
+  const esArchiver = getService('esArchiver');
+
+  describe('Reporting', function () {
+    this.tags(['smoke', 'ciGroup2']);
+    before(async () => {
+      await esArchiver.loadIfNeeded('packaging');
+    });
+
+    after(async () => {
+      await esArchiver.unload('packaging');
+      await es.deleteByQuery({
+        index: '.reporting-*',
+        refresh: true,
+        body: { query: { match_all: {} } },
+      });
+    });
+
+    it('downloaded PDF has OK status', async function () {
+      this.timeout(180000);
+
+      await pageObjects.common.navigateToApp('dashboards');
+      await pageObjects.dashboard.loadSavedDashboard('dashboard');
+      await pageObjects.reporting.openPdfReportingPanel();
+      await pageObjects.reporting.clickGenerateReportButton();
+
+      const url = await pageObjects.reporting.getReportURL(60000);
+      const res = await pageObjects.reporting.getResponse(url);
+
+      expect(res.status).to.equal(200);
+      expect(res.get('content-type')).to.equal('application/pdf');
+    });
+  });
+}

x-pack/test/functional/config.js

@@ -59,6 +59,7 @@ export default async function ({ readConfigFile }) {
       resolve(__dirname, './apps/transform'),
       resolve(__dirname, './apps/reporting_management'),
       resolve(__dirname, './apps/management'),
+      resolve(__dirname, './apps/reporting'),
 
       // This license_management file must be last because it is destructive.
       resolve(__dirname, './apps/license_management'),