[APM] E2E: Zero config for running e2e locally (#59152) (#61146)

2020-03-24 22:38:27 +01:00 · 2020-03-24 22:38:27 +01:00 · ad57ddb442
commit ad57ddb442
parent b5e271f126
17 changed files with 850 additions and 685 deletions
--- a/.gitignore
+++ b/.gitignore
@ -45,3 +45,6 @@ package-lock.json
 *.sublime-*
 npm-debug.log*
 .tern-project
+x-pack/legacy/plugins/apm/tsconfig.json
+apm.tsconfig.json
+/x-pack/legacy/plugins/apm/e2e/snapshots.js
--- a/x-pack/legacy/plugins/apm/e2e/.gitignore
+++ b/x-pack/legacy/plugins/apm/e2e/.gitignore
@ -1,4 +1,3 @@
-cypress/ingest-data/events.json
 cypress/screenshots/*
-
 cypress/test-results
+tmp
--- a/x-pack/legacy/plugins/apm/e2e/README.md
+++ b/x-pack/legacy/plugins/apm/e2e/README.md
@ -1,58 +1,16 @@
 # End-To-End (e2e) Test for APM UI

-## Ingest static data into Elasticsearch via APM Server
+**Run E2E tests**

-1. Start Elasticsearch and APM Server, using [apm-integration-testing](https://github.com/elastic/apm-integration-testing):
-
-```shell
-$ git clone https://github.com/elastic/apm-integration-testing.git
-$ cd apm-integration-testing
-./scripts/compose.py start master --no-kibana --no-xpack-secure
+```sh
+x-pack/legacy/plugins/apm/e2e/run-e2e.sh
 ```

-2. Download [static data file](https://storage.googleapis.com/apm-ui-e2e-static-data/events.json)
-
-```shell
-$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
-$ curl https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output events.json
-```
-
-3. Post to APM Server
-
-```shell
-$ cd x-pack/legacy/plugins/apm/e2e/cypress/ingest-data
-$ node replay.js --server-url http://localhost:8200 --secret-token abcd --events ./events.json
-```
->This process will take a few minutes to ingest all data
-
-4. Start Kibana
-
-```shell
-$ yarn kbn bootstrap
-$ yarn start --no-base-path --csp.strict=false
-```
-
-> Content Security Policy (CSP) Settings: Your Kibana instance must have the `csp.strict: false`.
-
-## How to run the tests
-
-_Note: Run the following commands from `kibana/x-pack/legacy/plugins/apm/e2e/cypress`._
-
-### Interactive mode
-
-```
-yarn cypress open
-```
-
-### Headless mode
-
-```
-yarn cypress run
-```
+_Starts Kibana, APM Server, Elasticsearch (with sample data) and runs the tests_

 ## Reproducing CI builds

->This process is very slow compared to the local development described above. Consider that the CI must install and configure the build tools and create a Docker image for the project to run tests in a consistent manner.
+> This process is very slow compared to the local development described above. Consider that the CI must install and configure the build tools and create a Docker image for the project to run tests in a consistent manner.

 The Jenkins CI uses a shell script to prepare Kibana:

@ -60,7 +18,7 @@ The Jenkins CI uses a shell script to prepare Kibana:
 # Prepare and run Kibana locally
 $ x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
 # Build Docker image for Kibana
-$ docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) x-pack/legacy/plugins/apm/e2e/ci 
+$ docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) x-pack/legacy/plugins/apm/e2e/ci
 # Run Docker image
 $ docker run --rm -t --user "$(id -u):$(id -g)" \
    -v `pwd`:/app --network="host" \
--- a/x-pack/legacy/plugins/apm/e2e/ci/entrypoint.sh
+++ b/x-pack/legacy/plugins/apm/e2e/ci/entrypoint.sh
@ -7,7 +7,7 @@ if [ -z "${kibana}" ] ; then
  kibana=127.0.0.1
 fi

-export CYPRESS_BASE_URL=http://${kibana}:5601
+export CYPRESS_BASE_URL=http://${kibana}:5701

 ## To avoid issues with the home and caching artifacts
 export HOME=/tmp
--- a/x-pack/legacy/plugins/apm/e2e/ci/kibana.dev.yml
+++ b/x-pack/legacy/plugins/apm/e2e/ci/kibana.dev.yml
@ -1,7 +0,0 @@
-##
-# Disabled plugins
-########################
-logging.verbose: true
-elasticsearch.username: "kibana_system_user"
-elasticsearch.password: "changeme"
-xpack.security.encryptionKey: "something_at_least_32_characters"
--- a/x-pack/legacy/plugins/apm/e2e/ci/kibana.e2e.yml
+++ b/x-pack/legacy/plugins/apm/e2e/ci/kibana.e2e.yml
@ -0,0 +1,31 @@
+# Kibana
+server.port: 5701
+xpack.security.encryptionKey: 'something_at_least_32_characters'
+csp.strict: false
+logging.verbose: true
+
+# Elasticsearch
+# Started via apm-integration-testing
+# ./scripts/compose.py start master --no-kibana --elasticsearch-port 9201 --apm-server-port 8201
+elasticsearch.hosts: http://localhost:9201
+elasticsearch.username: 'kibana_system_user'
+elasticsearch.password: 'changeme'
+
+# APM index pattern
+apm_oss.indexPattern: apm-*
+
+# APM Indices
+apm_oss.errorIndices: apm-*-error*
+apm_oss.sourcemapIndices: apm-*-sourcemap
+apm_oss.transactionIndices: apm-*-transaction*
+apm_oss.spanIndices: apm-*-span*
+apm_oss.metricsIndices: apm-*-metric*
+apm_oss.onboardingIndices: apm-*-onboarding*
+
+# APM options
+xpack.apm.enabled: true
+xpack.apm.serviceMapEnabled: false
+xpack.apm.autocreateApmIndexPattern: true
+xpack.apm.ui.enabled: true
+xpack.apm.ui.transactionGroupBucketSize: 100
+xpack.apm.ui.maxTraceItems: 1000
--- a/x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
+++ b/x-pack/legacy/plugins/apm/e2e/ci/prepare-kibana.sh
@ -1,24 +1,21 @@
 #!/usr/bin/env bash
 set -e

-CYPRESS_DIR="x-pack/legacy/plugins/apm/e2e"
+E2E_DIR="x-pack/legacy/plugins/apm/e2e"

 echo "1/3 Install dependencies ..."
 # shellcheck disable=SC1091
 source src/dev/ci_setup/setup_env.sh true
 yarn kbn bootstrap
-cp ${CYPRESS_DIR}/ci/kibana.dev.yml config/kibana.dev.yml
-echo 'elasticsearch:' >> config/kibana.dev.yml
-cp ${CYPRESS_DIR}/ci/kibana.dev.yml config/kibana.yml

 echo "2/3 Ingest test data ..."
-pushd ${CYPRESS_DIR}
+pushd ${E2E_DIR}
 yarn install
 curl --silent https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output ingest-data/events.json
-node ingest-data/replay.js --server-url http://localhost:8200 --secret-token abcd --events ./events.json > ingest-data.log
+node ingest-data/replay.js --server-url http://localhost:8201 --secret-token abcd --events ./events.json > ingest-data.log

 echo "3/3 Start Kibana ..."
 popd
 ## Might help to avoid FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory
 export NODE_OPTIONS="--max-old-space-size=4096"
-nohup node scripts/kibana --no-base-path --csp.strict=false --optimize.watch=false> kibana.log 2>&1 &
+nohup node scripts/kibana --config "${E2E_DIR}/ci/kibana.e2e.yml" --no-base-path --optimize.watch=false> kibana.log 2>&1 &
--- a/x-pack/legacy/plugins/apm/e2e/cypress.json
+++ b/x-pack/legacy/plugins/apm/e2e/cypress.json
@ -1,5 +1,6 @@
 {
-  "baseUrl": "http://localhost:5601",
+  "nodeVersion": "system",
+  "baseUrl": "http://localhost:5701",
  "video": false,
  "trashAssetsBeforeRuns": false,
  "fileServerFolder": "../",
@ -15,5 +16,9 @@
    "mochaFile": "./cypress/test-results/[hash]-e2e-tests.xml",
    "toConsole": false
  },
-  "testFiles": "**/*.{feature,features}"
+  "testFiles": "**/*.{feature,features}",
+  "env": {
+    "elasticsearch_username": "admin",
+    "elasticsearch_password": "changeme"
+  }
 }
--- a/x-pack/legacy/plugins/apm/e2e/cypress/integration/apm.feature
+++ b/x-pack/legacy/plugins/apm/e2e/cypress/integration/apm.feature
@ -2,6 +2,6 @@ Feature: APM

  Scenario: Transaction duration charts
    Given a user browses the APM UI application
-    When the user inspects the opbeans-go service
+    When the user inspects the opbeans-node service
    Then should redirect to correct path with correct params
-      And should have correct y-axis ticks
+      And should have correct y-axis ticks
--- a/x-pack/legacy/plugins/apm/e2e/cypress/integration/helpers.ts
+++ b/x-pack/legacy/plugins/apm/e2e/cypress/integration/helpers.ts
@ -6,45 +6,26 @@

 /* eslint-disable import/no-extraneous-dependencies */

-import { safeLoad } from 'js-yaml';
-
-const RANGE_FROM = '2019-09-04T18:00:00.000Z';
-const RANGE_TO = '2019-09-05T06:00:00.000Z';
+const RANGE_FROM = '2020-03-04T12:30:00.000Z';
+const RANGE_TO = '2020-03-04T13:00:00.000Z';
 const BASE_URL = Cypress.config().baseUrl;

-/**
- * Credentials in the `kibana.dev.yml` config file will be used to authenticate with Kibana
- */
-const KIBANA_DEV_YML_PATH = '../../../../../config/kibana.dev.yml';
-
 /** The default time in ms to wait for a Cypress command to complete */
-export const DEFAULT_TIMEOUT = 30 * 1000;
+export const DEFAULT_TIMEOUT = 60 * 1000;

 export function loginAndWaitForPage(url: string) {
-  // read the login details from `kibana.dev.yml`
-  cy.readFile(KIBANA_DEV_YML_PATH).then(kibanaDevYml => {
-    const config = safeLoad(kibanaDevYml);
-    const username = config['elasticsearch.username'];
-    const password = config['elasticsearch.password'];
+  const username = Cypress.env('elasticsearch_username');
+  const password = Cypress.env('elasticsearch_password');

-    const hasCredentials = username && password;
+  cy.log(`Authenticating via ${username} / ${password}`);

-    cy.log(
-      `Authenticating via config credentials from "${KIBANA_DEV_YML_PATH}". username: ${username}, password: ${password}`
-    );
-
-    const options = hasCredentials
-      ? {
-          auth: { username, password }
-        }
-      : {};
-
-    const fullUrl = `${BASE_URL}${url}?rangeFrom=${RANGE_FROM}&rangeTo=${RANGE_TO}`;
-    cy.visit(fullUrl, options);
-  });
+  const fullUrl = `${BASE_URL}${url}?rangeFrom=${RANGE_FROM}&rangeTo=${RANGE_TO}`;
+  cy.visit(fullUrl, { auth: { username, password } });

  cy.viewport('macbook-15');

  // wait for loading spinner to disappear
-  cy.get('.kibanaLoaderWrap', { timeout: DEFAULT_TIMEOUT }).should('not.exist');
+  cy.get('#kbn_loading_message', { timeout: DEFAULT_TIMEOUT }).should(
+    'not.exist'
+  );
 }
--- a/x-pack/legacy/plugins/apm/e2e/cypress/integration/snapshots.js
+++ b/x-pack/legacy/plugins/apm/e2e/cypress/integration/snapshots.js
@ -1,19 +1,10 @@
 module.exports = {
-  "When clicking opbeans-go service": {
-    "transaction duration charts": {
-      "should have correct y-axis ticks": {
-        "1": "3.7 min",
-        "2": "1.8 min",
-        "3": "0.0 min"
-      }
-    }
-  },
-  "__version": "3.8.3",
  "APM": {
    "Transaction duration charts": {
-      "1": "3.7 min",
-      "2": "1.8 min",
-      "3": "0.0 min"
+      "1": "500 ms",
+      "2": "250 ms",
+      "3": "0 ms"
    }
-  }
+  },
+  "__version": "4.2.0"
 }
--- a/x-pack/legacy/plugins/apm/e2e/cypress/support/step_definitions/apm.ts
+++ b/x-pack/legacy/plugins/apm/e2e/cypress/support/step_definitions/apm.ts
@ -12,15 +12,15 @@ Given(`a user browses the APM UI application`, () => {
  loginAndWaitForPage(`/app/apm#/services`);
 });

-When(`the user inspects the opbeans-go service`, () => {
-  // click opbeans-go service
-  cy.get(':contains(opbeans-go)')
+When(`the user inspects the opbeans-node service`, () => {
+  // click opbeans-node service
+  cy.get(':contains(opbeans-node)')
    .last()
    .click({ force: true });
 });

 Then(`should redirect to correct path with correct params`, () => {
-  cy.url().should('contain', `/app/apm#/services/opbeans-go/transactions`);
+  cy.url().should('contain', `/app/apm#/services/opbeans-node/transactions`);
  cy.url().should('contain', `transactionType=request`);
 });

--- a/x-pack/legacy/plugins/apm/e2e/cypress/ingest-data/replay.js
+++ b/x-pack/legacy/plugins/apm/e2e/cypress/ingest-data/replay.js
@ -4,15 +4,21 @@
 * you may not use this file except in compliance with the Elastic License.
 */

-/* eslint-disable import/no-extraneous-dependencies */
 /* eslint-disable no-console */

+/* eslint-disable import/no-extraneous-dependencies */
+
 /**
 * This script is useful for ingesting previously generated APM data into Elasticsearch via APM Server
 *
 * You can either:
 *  1. Download a static test data file from: https://storage.googleapis.com/apm-ui-e2e-static-data/events.json
- *  2. Or, generate the test data file yourself by following the steps in: https://github.com/sqren/kibana/blob/master/x-pack/legacy/plugins/apm/cypress/README.md#how-to-generate-static-data
+ *  2. Or, generate the test data file yourself:
+ *    git clone https://github.com/elastic/apm-integration-testing.git
+ *    ./scripts/compose.py start master --no-kibana --with-opbeans-node --apm-server-record
+ *    docker cp localtesting_8.0.0_apm-server-2:/app/events.json . && cat events.json | wc -l
+ *
+ *
 *
 * Run the script:
 *
@ -27,6 +33,7 @@ const axios = require('axios');
 const readFile = promisify(fs.readFile);
 const pLimit = require('p-limit');
 const { argv } = require('yargs');
+const ora = require('ora');

 const APM_SERVER_URL = argv.serverUrl;
 const SECRET_TOKEN = argv.secretToken;
@ -43,10 +50,27 @@ if (!EVENTS_PATH) {
 }

 const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+const requestProgress = {
+  succeeded: 0,
+  failed: 0,
+  total: 0
+};
+
+const spinner = ora({ text: 'Warming up...', stream: process.stdout });
+
+function updateSpinnerText({ success }) {
+  success ? requestProgress.succeeded++ : requestProgress.failed++;
+  const remaining =
+    requestProgress.total -
+    (requestProgress.succeeded + requestProgress.failed);
+
+  spinner.text = `Remaining: ${remaining}. Succeeded: ${requestProgress.succeeded}. Failed: ${requestProgress.failed}.`;
+}
+
 async function insertItem(item) {
  try {
    const url = `${APM_SERVER_URL}${item.url}`;
-    console.log(Date.now(), url);

    const headers = {
      'content-type': 'application/x-ndjson'
@ -63,20 +87,20 @@ async function insertItem(item) {
      data: item.body
    });

+    updateSpinnerText({ success: true });
+
    // add delay to avoid flooding the queue
    return delay(500);
  } catch (e) {
-    console.log('an error occurred');
-    if (e.response) {
-      console.log(e.response.data);
-    } else {
-      console.log('error', e);
-    }
+    console.error(
+      `${e.response ? JSON.stringify(e.response.data) : e.message}`
+    );
+    updateSpinnerText({ success: false });
  }
 }

 async function init() {
-  const content = await readFile(path.resolve(__dirname, EVENTS_PATH));
+  const content = await readFile(path.resolve(EVENTS_PATH));
  const items = content
    .toString()
    .split('\n')
@ -84,10 +108,21 @@ async function init() {
    .map(item => JSON.parse(item))
    .filter(item => item.url === '/intake/v2/events');

+  spinner.start();
+  requestProgress.total = items.length;
+
  const limit = pLimit(20); // number of concurrent requests
  await Promise.all(items.map(item => limit(() => insertItem(item))));
 }

-init().catch(e => {
-  console.log('An error occurred:', e);
-});
+init()
+  .catch(e => {
+    console.log('An error occurred:', e);
+    process.exit(1);
+  })
+  .then(() => {
+    spinner.succeed(
+      `Successfully ingested ${requestProgress.succeeded} of ${requestProgress.total} events`
+    );
+    process.exit(0);
+  });
--- a/x-pack/legacy/plugins/apm/e2e/package.json
+++ b/x-pack/legacy/plugins/apm/e2e/package.json
@ -9,16 +9,18 @@
  },
  "dependencies": {
    "@cypress/snapshot": "^2.1.3",
-    "@cypress/webpack-preprocessor": "^4.1.0",
-    "@types/cypress-cucumber-preprocessor": "^1.14.0",
+    "@cypress/webpack-preprocessor": "^4.1.3",
+    "@types/cypress-cucumber-preprocessor": "^1.14.1",
    "@types/js-yaml": "^3.12.1",
    "@types/node": "^10.12.11",
-    "cypress": "^3.5.0",
+    "cypress": "^4.2.0",
    "cypress-cucumber-preprocessor": "^2.0.1",
    "js-yaml": "^3.13.1",
+    "ora": "^4.0.3",
    "p-limit": "^2.2.1",
-    "ts-loader": "^6.1.0",
-    "typescript": "3.7.5",
-    "webpack": "^4.41.5"
+    "ts-loader": "^6.2.2",
+    "typescript": "3.8.3",
+    "wait-on": "^4.0.1",
+    "webpack": "^4.42.1"
  }
 }
--- a/x-pack/legacy/plugins/apm/e2e/run-e2e.sh
+++ b/x-pack/legacy/plugins/apm/e2e/run-e2e.sh
@ -0,0 +1,93 @@
+# variables
+KIBANA_PORT=5701
+ELASTICSEARCH_PORT=9201
+APM_SERVER_PORT=8201
+
+# ensure Docker is running
+docker ps &> /dev/null
+if [ $? -ne 0 ]; then
+    echo "⚠️  Please start Docker"
+    exit 1
+fi
+
+# formatting
+bold=$(tput bold)
+normal=$(tput sgr0)
+
+# Create tmp folder
+mkdir -p tmp
+
+# Ask user to start Kibana
+echo "
+${bold}To start Kibana please run the following command:${normal}
+
+node ./scripts/kibana --no-base-path --dev --no-dev-config --config x-pack/legacy/plugins/apm/e2e/ci/kibana.e2e.yml
+"
+
+# Clone or pull apm-integration-testing
+printf "\n${bold}=== apm-integration-testing ===\n${normal}"
+
+git clone "https://github.com/elastic/apm-integration-testing.git" "./tmp/apm-integration-testing" &> /dev/null
+if [ $? -eq 0 ]; then
+    echo "Cloning repository"
+else
+    echo "Pulling from master..."
+    git -C "./tmp/apm-integration-testing" pull &> /dev/null
+fi
+
+# Start apm-integration-testing
+echo "Starting (logs: ./tmp/apm-it.log)"
+./tmp/apm-integration-testing/scripts/compose.py start master \
+    --no-kibana \
+    --elasticsearch-port $ELASTICSEARCH_PORT \
+    --apm-server-port=$APM_SERVER_PORT \
+    --elasticsearch-heap 4g \
+    &> ./tmp/apm-it.log
+
+# Stop if apm-integration-testing failed to start correctly
+if [ $? -ne 0 ]; then
+    printf "⚠️  apm-integration-testing could not be started.\n"
+    printf "Please see the logs in ./tmp/apm-it.log\n\n"
+    printf "As a last resort, reset docker with:\n\n./tmp/apm-integration-testing/scripts/compose.py stop && system prune --all --force --volumes\n"
+    exit 1
+fi
+
+printf "\n${bold}=== Static mock data ===\n${normal}"
+
+# Download static data if not already done
+if [ -e "./tmp/events.json" ]; then
+    echo 'Skip: events.json already exists. Not downloading'
+else
+    echo 'Downloading events.json...'
+    curl --silent https://storage.googleapis.com/apm-ui-e2e-static-data/events.json --output ./tmp/events.json
+fi
+
+# echo "Deleting existing indices (apm* and .apm*)"
+curl --silent --user admin:changeme -XDELETE "localhost:${ELASTICSEARCH_PORT}/.apm*" > /dev/null
+curl --silent --user admin:changeme -XDELETE "localhost:${ELASTICSEARCH_PORT}/apm*" > /dev/null
+
+# Ingest data into APM Server
+echo "Ingesting data (logs: tmp/ingest-data.log)"
+node ingest-data/replay.js --server-url http://localhost:$APM_SERVER_PORT --events ./tmp/events.json  2> ./tmp/ingest-data.log
+
+# Install local dependencies
+printf "\n"
+echo "Installing local dependencies (logs: tmp/e2e-yarn.log)"
+yarn &> ./tmp/e2e-yarn.log
+
+# Wait for Kibana to start
+echo "Waiting for Kibana to start..."
+yarn wait-on -i 500 -w 500 http://localhost:$KIBANA_PORT > /dev/null
+
+echo "\n✅ Setup completed successfully. Running tests...\n"
+
+# run cypress tests
+yarn cypress run --config pageLoadTimeout=100000,watchForFileChanges=true
+
+echo "
+
+${bold}If you want to run the test interactively, run:${normal}
+
+yarn cypress open --config pageLoadTimeout=100000,watchForFileChanges=true
+"
+
--- a/x-pack/legacy/plugins/apm/e2e/tsconfig.json
+++ b/x-pack/legacy/plugins/apm/e2e/tsconfig.json
@ -1,13 +1,8 @@
 {
  "extends": "../../../../tsconfig.json",
-  "exclude": [],
-  "include": [
-    "./**/*"
-  ],
+  "exclude": ["tmp"],
+  "include": ["./**/*"],
  "compilerOptions": {
-    "types": [
-      "cypress",
-      "node"
-    ]
+    "types": ["cypress", "node"]
  }
 }
--- a/x-pack/legacy/plugins/apm/e2e/yarn.lock
+++ b/x-pack/legacy/plugins/apm/e2e/yarn.lock