From 6de7f2a62b2b078b703bbe6f18475909e1224f57 Mon Sep 17 00:00:00 2001 From: spalger Date: Mon, 23 Mar 2020 22:26:15 -0700 Subject: [PATCH] Revert "[APM] Collect telemetry about data/API performance (#51612)" This reverts commit 13baa5156151af258249afc6468f15b53fffeee0. --- src/dev/run_check_lockfile_symlinks.js | 2 - x-pack/legacy/plugins/apm/index.ts | 19 +- x-pack/legacy/plugins/apm/mappings.json | 783 +----------------- x-pack/legacy/plugins/apm/scripts/.gitignore | 1 - .../legacy/plugins/apm/scripts/package.json | 10 - .../apm/scripts/setup-kibana-security.js | 1 - .../apm/scripts/upload-telemetry-data.js | 21 - .../download-telemetry-template.ts | 26 - .../generate-sample-documents.ts | 124 --- .../scripts/upload-telemetry-data/index.ts | 208 ----- .../elasticsearch_fieldnames.test.ts.snap | 48 +- x-pack/plugins/apm/common/agent_name.ts | 44 +- .../apm/common/apm_saved_object_constants.ts | 10 +- .../common/elasticsearch_fieldnames.test.ts | 15 +- .../apm/common/elasticsearch_fieldnames.ts | 11 +- x-pack/plugins/apm/kibana.json | 7 +- x-pack/plugins/apm/server/index.ts | 6 +- .../lib/apm_telemetry/__test__/index.test.ts | 83 ++ .../collect_data_telemetry/index.ts | 77 -- .../collect_data_telemetry/tasks.ts | 725 ---------------- .../apm/server/lib/apm_telemetry/index.ts | 153 +--- .../apm/server/lib/apm_telemetry/types.ts | 118 --- .../server/lib/helpers/setup_request.test.ts | 13 - x-pack/plugins/apm/server/plugin.ts | 37 +- .../server/routes/create_api/index.test.ts | 1 - x-pack/plugins/apm/server/routes/services.ts | 16 + .../apm/typings/elasticsearch/aggregations.ts | 26 +- .../apm/typings/es_schemas/raw/error_raw.ts | 2 - .../typings/es_schemas/raw/fields/observer.ts | 10 - .../apm/typings/es_schemas/raw/span_raw.ts | 2 - .../typings/es_schemas/raw/transaction_raw.ts | 2 - 31 files changed, 210 insertions(+), 2391 deletions(-) delete mode 100644 x-pack/legacy/plugins/apm/scripts/.gitignore delete mode 100644 x-pack/legacy/plugins/apm/scripts/package.json delete mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js delete mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts delete mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts delete mode 100644 x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts create mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts delete mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts delete mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts delete mode 100644 x-pack/plugins/apm/server/lib/apm_telemetry/types.ts delete mode 100644 x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts diff --git a/src/dev/run_check_lockfile_symlinks.js b/src/dev/run_check_lockfile_symlinks.js index 6c6fc54638ee..54a8cdf638a7 100644 --- a/src/dev/run_check_lockfile_symlinks.js +++ b/src/dev/run_check_lockfile_symlinks.js @@ -36,8 +36,6 @@ const IGNORE_FILE_GLOBS = [ '**/*fixtures*/**/*', // cypress isn't used in production, ignore it 'x-pack/legacy/plugins/apm/e2e/*', - // apm scripts aren't used in production, ignore them - 'x-pack/legacy/plugins/apm/scripts/*', ]; run(async ({ log }) => { diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 594e8a4a7af7..0107997f233f 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -14,13 +14,7 @@ import mappings from './mappings.json'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: [ - 'kibana', - 'elasticsearch', - 'xpack_main', - 'apm_oss', - 'task_manager' - ], + require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), @@ -77,10 +71,7 @@ export const apm: LegacyPluginInitializer = kibana => { autocreateApmIndexPattern: Joi.boolean().default(true), // service map - serviceMapEnabled: Joi.boolean().default(true), - - // telemetry - telemetryCollectionEnabled: Joi.boolean().default(true) + serviceMapEnabled: Joi.boolean().default(true) }).default(); }, @@ -116,12 +107,10 @@ export const apm: LegacyPluginInitializer = kibana => { } } }); + const apmPlugin = server.newPlatform.setup.plugins .apm as APMPluginContract; - - apmPlugin.registerLegacyAPI({ - server - }); + apmPlugin.registerLegacyAPI({ server }); } }); }; diff --git a/x-pack/legacy/plugins/apm/mappings.json b/x-pack/legacy/plugins/apm/mappings.json index ba4c7a89ceaa..61bc90da2875 100644 --- a/x-pack/legacy/plugins/apm/mappings.json +++ b/x-pack/legacy/plugins/apm/mappings.json @@ -1,655 +1,12 @@ { - "apm-telemetry": { + "apm-services-telemetry": { "properties": { - "agents": { - "properties": { - "dotnet": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "language": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "go": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "java": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "type": "object" - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "js-base": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "type": "object" - }, - "language": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "type": "object" - } - } - } - } - }, - "nodejs": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "language": { - "properties": { - "name": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "python": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "ruby": { - "properties": { - "agent": { - "properties": { - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "service": { - "properties": { - "framework": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "language": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - }, - "runtime": { - "properties": { - "composite": { - "type": "keyword", - "ignore_above": 256 - }, - "name": { - "type": "keyword", - "ignore_above": 256 - }, - "version": { - "type": "keyword", - "ignore_above": 256 - } - } - } - } - } - } - }, - "rum-js": { - "properties": { - "agent": { - "type": "object" - }, - "service": { - "properties": { - "framework": { - "type": "object" - }, - "language": { - "type": "object" - }, - "runtime": { - "type": "object" - } - } - } - } - } - } - }, - "counts": { - "properties": { - "agent_configuration": { - "properties": { - "all": { - "type": "long" - } - } - }, - "error": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "max_error_groups_per_service": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "max_transaction_groups_per_service": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "metric": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "onboarding": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "services": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "sourcemap": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "span": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - }, - "traces": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "transaction": { - "properties": { - "1d": { - "type": "long" - }, - "all": { - "type": "long" - } - } - } - } - }, - "cardinality": { - "properties": { - "user_agent": { - "properties": { - "original": { - "properties": { - "all_agents": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "rum": { - "properties": { - "1d": { - "type": "long" - } - } - } - } - } - } - }, - "transaction": { - "properties": { - "name": { - "properties": { - "all_agents": { - "properties": { - "1d": { - "type": "long" - } - } - }, - "rum": { - "properties": { - "1d": { - "type": "long" - } - } - } - } - } - } - } - } - }, "has_any_services": { "type": "boolean" }, - "indices": { - "properties": { - "all": { - "properties": { - "total": { - "properties": { - "docs": { - "properties": { - "count": { - "type": "long" - } - } - }, - "store": { - "properties": { - "size_in_bytes": { - "type": "long" - } - } - } - } - } - } - }, - "shards": { - "properties": { - "total": { - "type": "long" - } - } - } - } - }, - "integrations": { - "properties": { - "ml": { - "properties": { - "all_jobs_count": { - "type": "long" - } - } - } - } - }, - "retainment": { - "properties": { - "error": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "metric": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "onboarding": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "span": { - "properties": { - "ms": { - "type": "long" - } - } - }, - "transaction": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, "services_per_agent": { "properties": { - "dotnet": { - "type": "long", - "null_value": 0 - }, - "go": { + "python": { "type": "long", "null_value": 0 }, @@ -657,15 +14,19 @@ "type": "long", "null_value": 0 }, - "js-base": { - "type": "long", - "null_value": 0 - }, "nodejs": { "type": "long", "null_value": 0 }, - "python": { + "js-base": { + "type": "long", + "null_value": 0 + }, + "rum-js": { + "type": "long", + "null_value": 0 + }, + "dotnet": { "type": "long", "null_value": 0 }, @@ -673,131 +34,11 @@ "type": "long", "null_value": 0 }, - "rum-js": { + "go": { "type": "long", "null_value": 0 } } - }, - "tasks": { - "properties": { - "agent_configuration": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "agents": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "cardinality": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "groupings": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "indices_stats": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "integrations": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "processor_events": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "services": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - }, - "versions": { - "properties": { - "took": { - "properties": { - "ms": { - "type": "long" - } - } - } - } - } - } - }, - "version": { - "properties": { - "apm_server": { - "properties": { - "major": { - "type": "long" - }, - "minor": { - "type": "long" - }, - "patch": { - "type": "long" - } - } - } - } } } }, diff --git a/x-pack/legacy/plugins/apm/scripts/.gitignore b/x-pack/legacy/plugins/apm/scripts/.gitignore deleted file mode 100644 index 8ee01d321b72..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/.gitignore +++ /dev/null @@ -1 +0,0 @@ -yarn.lock diff --git a/x-pack/legacy/plugins/apm/scripts/package.json b/x-pack/legacy/plugins/apm/scripts/package.json deleted file mode 100644 index 9121449c5361..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "apm-scripts", - "version": "1.0.0", - "main": "index.js", - "license": "MIT", - "dependencies": { - "@octokit/rest": "^16.35.0", - "console-stamp": "^0.2.9" - } -} diff --git a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js index 61ba2fdc7f7e..825c1a526fcc 100644 --- a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js +++ b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js @@ -16,7 +16,6 @@ ******************************/ // compile typescript on the fly -// eslint-disable-next-line import/no-extraneous-dependencies require('@babel/register')({ extensions: ['.ts'], plugins: ['@babel/plugin-proposal-optional-chaining'], diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js deleted file mode 100644 index a99651c62dd7..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// compile typescript on the fly -// eslint-disable-next-line import/no-extraneous-dependencies -require('@babel/register')({ - extensions: ['.ts'], - plugins: [ - '@babel/plugin-proposal-optional-chaining', - '@babel/plugin-proposal-nullish-coalescing-operator' - ], - presets: [ - '@babel/typescript', - ['@babel/preset-env', { targets: { node: 'current' } }] - ] -}); - -require('./upload-telemetry-data/index.ts'); diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts deleted file mode 100644 index dfed9223ef70..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-template.ts +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// @ts-ignore -import { Octokit } from '@octokit/rest'; - -export async function downloadTelemetryTemplate(octokit: Octokit) { - const file = await octokit.repos.getContents({ - owner: 'elastic', - repo: 'telemetry', - path: 'config/templates/xpack-phone-home.json', - // @ts-ignore - mediaType: { - format: 'application/vnd.github.VERSION.raw' - } - }); - - if (Array.isArray(file.data)) { - throw new Error('Expected single response, got array'); - } - - return JSON.parse(Buffer.from(file.data.content!, 'base64').toString()); -} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts deleted file mode 100644 index 8d76063a7fdf..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { DeepPartial } from 'utility-types'; -import { - merge, - omit, - defaultsDeep, - range, - mapValues, - isPlainObject, - flatten -} from 'lodash'; -import uuid from 'uuid'; -import { - CollectTelemetryParams, - collectDataTelemetry - // eslint-disable-next-line @kbn/eslint/no-restricted-paths -} from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry'; - -interface GenerateOptions { - days: number; - instances: number; - variation: { - min: number; - max: number; - }; -} - -const randomize = ( - value: unknown, - instanceVariation: number, - dailyGrowth: number -) => { - if (typeof value === 'boolean') { - return Math.random() > 0.5; - } - if (typeof value === 'number') { - return Math.round(instanceVariation * dailyGrowth * value); - } - return value; -}; - -const mapValuesDeep = ( - obj: Record, - iterator: (value: unknown, key: string, obj: Record) => unknown -): Record => - mapValues(obj, (val, key) => - isPlainObject(val) ? mapValuesDeep(val, iterator) : iterator(val, key!, obj) - ); - -export async function generateSampleDocuments( - options: DeepPartial & { - collectTelemetryParams: CollectTelemetryParams; - } -) { - const { collectTelemetryParams, ...preferredOptions } = options; - - const opts: GenerateOptions = defaultsDeep( - { - days: 100, - instances: 50, - variation: { - min: 0.1, - max: 4 - } - }, - preferredOptions - ); - - const sample = await collectDataTelemetry(collectTelemetryParams); - - console.log('Collected telemetry'); // eslint-disable-line no-console - console.log('\n' + JSON.stringify(sample, null, 2)); // eslint-disable-line no-console - - const dateOfScriptExecution = new Date(); - - return flatten( - range(0, opts.instances).map(instanceNo => { - const instanceId = uuid.v4(); - const defaults = { - cluster_uuid: instanceId, - stack_stats: { - kibana: { - versions: { - version: '8.0.0' - } - } - } - }; - - const instanceVariation = - Math.random() * (opts.variation.max - opts.variation.min) + - opts.variation.min; - - return range(0, opts.days).map(dayNo => { - const dailyGrowth = Math.pow(1.005, opts.days - 1 - dayNo); - - const timestamp = Date.UTC( - dateOfScriptExecution.getFullYear(), - dateOfScriptExecution.getMonth(), - -dayNo - ); - - const generated = mapValuesDeep(omit(sample, 'versions'), value => - randomize(value, instanceVariation, dailyGrowth) - ); - - return merge({}, defaults, { - timestamp, - stack_stats: { - kibana: { - plugins: { - apm: merge({}, sample, generated) - } - } - } - }); - }); - }) - ); -} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts deleted file mode 100644 index bdc57eac412f..000000000000 --- a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -// This script downloads the telemetry mapping, runs the APM telemetry tasks, -// generates a bunch of randomized data based on the downloaded sample, -// and uploads it to a cluster of your choosing in the same format as it is -// stored in the telemetry cluster. Its purpose is twofold: -// - Easier testing of the telemetry tasks -// - Validate whether we can run the queries we want to on the telemetry data - -import fs from 'fs'; -import path from 'path'; -// @ts-ignore -import { Octokit } from '@octokit/rest'; -import { merge, chunk, flatten, pick, identity } from 'lodash'; -import axios from 'axios'; -import yaml from 'js-yaml'; -import { Client } from 'elasticsearch'; -import { argv } from 'yargs'; -import { promisify } from 'util'; -import { Logger } from 'kibana/server'; -// @ts-ignore -import consoleStamp from 'console-stamp'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { CollectTelemetryParams } from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry'; -import { downloadTelemetryTemplate } from './download-telemetry-template'; -import mapping from '../../mappings.json'; -import { generateSampleDocuments } from './generate-sample-documents'; - -consoleStamp(console, '[HH:MM:ss.l]'); - -const githubToken = process.env.GITHUB_TOKEN; - -if (!githubToken) { - throw new Error('GITHUB_TOKEN was not provided.'); -} - -const kibanaConfigDir = path.join(__filename, '../../../../../../../config'); -const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml'); -const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml'); - -const xpackTelemetryIndexName = 'xpack-phone-home'; - -const loadedKibanaConfig = (yaml.safeLoad( - fs.readFileSync( - fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, - 'utf8' - ) -) || {}) as {}; - -const cliEsCredentials = pick( - { - 'elasticsearch.username': process.env.ELASTICSEARCH_USERNAME, - 'elasticsearch.password': process.env.ELASTICSEARCH_PASSWORD, - 'elasticsearch.hosts': process.env.ELASTICSEARCH_HOST - }, - identity -) as { - 'elasticsearch.username': string; - 'elasticsearch.password': string; - 'elasticsearch.hosts': string; -}; - -const config = { - 'apm_oss.transactionIndices': 'apm-*', - 'apm_oss.metricsIndices': 'apm-*', - 'apm_oss.errorIndices': 'apm-*', - 'apm_oss.spanIndices': 'apm-*', - 'apm_oss.onboardingIndices': 'apm-*', - 'apm_oss.sourcemapIndices': 'apm-*', - 'elasticsearch.hosts': 'http://localhost:9200', - ...loadedKibanaConfig, - ...cliEsCredentials -}; - -async function uploadData() { - const octokit = new Octokit({ - auth: githubToken - }); - - const telemetryTemplate = await downloadTelemetryTemplate(octokit); - - const kibanaMapping = mapping['apm-telemetry']; - - const httpAuth = - config['elasticsearch.username'] && config['elasticsearch.password'] - ? { - username: config['elasticsearch.username'], - password: config['elasticsearch.password'] - } - : null; - - const client = new Client({ - host: config['elasticsearch.hosts'], - ...(httpAuth - ? { - httpAuth: `${httpAuth.username}:${httpAuth.password}` - } - : {}) - }); - - if (argv.clear) { - try { - await promisify(client.indices.delete.bind(client))({ - index: xpackTelemetryIndexName - }); - } catch (err) { - // 404 = index not found, totally okay - if (err.status !== 404) { - throw err; - } - } - } - - const axiosInstance = axios.create({ - baseURL: config['elasticsearch.hosts'], - ...(httpAuth ? { auth: httpAuth } : {}) - }); - - const newTemplate = merge(telemetryTemplate, { - settings: { - index: { mapping: { total_fields: { limit: 10000 } } } - } - }); - - // override apm mapping instead of merging - newTemplate.mappings.properties.stack_stats.properties.kibana.properties.plugins.properties.apm = kibanaMapping; - - await axiosInstance.put(`/_template/xpack-phone-home`, newTemplate); - - const sampleDocuments = await generateSampleDocuments({ - collectTelemetryParams: { - logger: (console as unknown) as Logger, - indices: { - ...config, - apmCustomLinkIndex: '.apm-custom-links', - apmAgentConfigurationIndex: '.apm-agent-configuration' - }, - search: body => { - return promisify(client.search.bind(client))({ - ...body, - requestTimeout: 120000 - }) as any; - }, - indicesStats: body => { - return promisify(client.indices.stats.bind(client))({ - ...body, - requestTimeout: 120000 - }) as any; - }, - transportRequest: (params => { - return axiosInstance[params.method](params.path); - }) as CollectTelemetryParams['transportRequest'] - } - }); - - const chunks = chunk(sampleDocuments, 250); - - await chunks.reduce>((prev, documents) => { - return prev.then(async () => { - const body = flatten( - documents.map(doc => [{ index: { _index: 'xpack-phone-home' } }, doc]) - ); - - return promisify(client.bulk.bind(client))({ - body, - refresh: true - }).then((response: any) => { - if (response.errors) { - const firstError = response.items.filter( - (item: any) => item.index.status >= 400 - )[0].index.error; - throw new Error(`Failed to upload documents: ${firstError.reason} `); - } - }); - }); - }, Promise.resolve()); -} - -uploadData() - .catch(e => { - if ('response' in e) { - if (typeof e.response === 'string') { - // eslint-disable-next-line no-console - console.log(e.response); - } else { - // eslint-disable-next-line no-console - console.log( - JSON.stringify( - e.response, - ['status', 'statusText', 'headers', 'data'], - 2 - ) - ); - } - } else { - // eslint-disable-next-line no-console - console.log(e); - } - process.exit(1); - }) - .then(() => { - // eslint-disable-next-line no-console - console.log('Finished uploading generated telemetry data'); - }); diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 5de82a9ee878..897d4e979fce 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -2,8 +2,6 @@ exports[`Error AGENT_NAME 1`] = `"java"`; -exports[`Error AGENT_VERSION 1`] = `"agent version"`; - exports[`Error CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Error CONTAINER_ID 1`] = `undefined`; @@ -58,7 +56,7 @@ exports[`Error METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; -exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `undefined`; exports[`Error PARENT_ID 1`] = `"parentId"`; @@ -70,20 +68,10 @@ exports[`Error SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Error SERVICE_FRAMEWORK_NAME 1`] = `undefined`; -exports[`Error SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; - -exports[`Error SERVICE_LANGUAGE_NAME 1`] = `"nodejs"`; - -exports[`Error SERVICE_LANGUAGE_VERSION 1`] = `"v1337"`; - exports[`Error SERVICE_NAME 1`] = `"service name"`; exports[`Error SERVICE_NODE_NAME 1`] = `undefined`; -exports[`Error SERVICE_RUNTIME_NAME 1`] = `undefined`; - -exports[`Error SERVICE_RUNTIME_VERSION 1`] = `undefined`; - exports[`Error SERVICE_VERSION 1`] = `undefined`; exports[`Error SPAN_ACTION 1`] = `undefined`; @@ -124,14 +112,10 @@ exports[`Error URL_FULL 1`] = `undefined`; exports[`Error USER_AGENT_NAME 1`] = `undefined`; -exports[`Error USER_AGENT_ORIGINAL 1`] = `undefined`; - exports[`Error USER_ID 1`] = `undefined`; exports[`Span AGENT_NAME 1`] = `"java"`; -exports[`Span AGENT_VERSION 1`] = `"agent version"`; - exports[`Span CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Span CONTAINER_ID 1`] = `undefined`; @@ -186,7 +170,7 @@ exports[`Span METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; -exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `undefined`; exports[`Span PARENT_ID 1`] = `"parentId"`; @@ -198,20 +182,10 @@ exports[`Span SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Span SERVICE_FRAMEWORK_NAME 1`] = `undefined`; -exports[`Span SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; - -exports[`Span SERVICE_LANGUAGE_NAME 1`] = `undefined`; - -exports[`Span SERVICE_LANGUAGE_VERSION 1`] = `undefined`; - exports[`Span SERVICE_NAME 1`] = `"service name"`; exports[`Span SERVICE_NODE_NAME 1`] = `undefined`; -exports[`Span SERVICE_RUNTIME_NAME 1`] = `undefined`; - -exports[`Span SERVICE_RUNTIME_VERSION 1`] = `undefined`; - exports[`Span SERVICE_VERSION 1`] = `undefined`; exports[`Span SPAN_ACTION 1`] = `"my action"`; @@ -252,14 +226,10 @@ exports[`Span URL_FULL 1`] = `undefined`; exports[`Span USER_AGENT_NAME 1`] = `undefined`; -exports[`Span USER_AGENT_ORIGINAL 1`] = `undefined`; - exports[`Span USER_ID 1`] = `undefined`; exports[`Transaction AGENT_NAME 1`] = `"java"`; -exports[`Transaction AGENT_VERSION 1`] = `"agent version"`; - exports[`Transaction CLIENT_GEO_COUNTRY_ISO_CODE 1`] = `undefined`; exports[`Transaction CONTAINER_ID 1`] = `"container1234567890abcdef"`; @@ -314,7 +284,7 @@ exports[`Transaction METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; -exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `undefined`; exports[`Transaction PARENT_ID 1`] = `"parentId"`; @@ -326,20 +296,10 @@ exports[`Transaction SERVICE_ENVIRONMENT 1`] = `undefined`; exports[`Transaction SERVICE_FRAMEWORK_NAME 1`] = `undefined`; -exports[`Transaction SERVICE_FRAMEWORK_VERSION 1`] = `undefined`; - -exports[`Transaction SERVICE_LANGUAGE_NAME 1`] = `"nodejs"`; - -exports[`Transaction SERVICE_LANGUAGE_VERSION 1`] = `"v1337"`; - exports[`Transaction SERVICE_NAME 1`] = `"service name"`; exports[`Transaction SERVICE_NODE_NAME 1`] = `undefined`; -exports[`Transaction SERVICE_RUNTIME_NAME 1`] = `undefined`; - -exports[`Transaction SERVICE_RUNTIME_VERSION 1`] = `undefined`; - exports[`Transaction SERVICE_VERSION 1`] = `undefined`; exports[`Transaction SPAN_ACTION 1`] = `undefined`; @@ -380,6 +340,4 @@ exports[`Transaction URL_FULL 1`] = `"http://www.elastic.co"`; exports[`Transaction USER_AGENT_NAME 1`] = `"Other"`; -exports[`Transaction USER_AGENT_ORIGINAL 1`] = `"test original"`; - exports[`Transaction USER_ID 1`] = `"1337"`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index 085828b729ea..bb68eb88b8e1 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -4,40 +4,36 @@ * you may not use this file except in compliance with the Elastic License. */ -import { AgentName } from '../typings/es_schemas/ui/fields/agent'; - /* * Agent names can be any string. This list only defines the official agents * that we might want to target specifically eg. linking to their documentation * & telemetry reporting. Support additional agent types by appending * definitions in mappings.json (for telemetry), the AgentName type, and the - * AGENT_NAMES array. + * agentNames object. */ +import { AgentName } from '../typings/es_schemas/ui/fields/agent'; -export const AGENT_NAMES: AgentName[] = [ - 'java', - 'js-base', - 'rum-js', - 'dotnet', - 'go', - 'java', - 'nodejs', - 'python', - 'ruby' -]; +const agentNames: { [agentName in AgentName]: agentName } = { + python: 'python', + java: 'java', + nodejs: 'nodejs', + 'js-base': 'js-base', + 'rum-js': 'rum-js', + dotnet: 'dotnet', + ruby: 'ruby', + go: 'go' +}; -export function isAgentName(agentName: string): agentName is AgentName { - return AGENT_NAMES.includes(agentName as AgentName); +export function isAgentName(agentName: string): boolean { + return Object.values(agentNames).includes(agentName as AgentName); } -export function isRumAgentName( - agentName: string | undefined -): agentName is 'js-base' | 'rum-js' { - return agentName === 'js-base' || agentName === 'rum-js'; +export function isRumAgentName(agentName: string | undefined) { + return ( + agentName === agentNames['js-base'] || agentName === agentNames['rum-js'] + ); } -export function isJavaAgentName( - agentName: string | undefined -): agentName is 'java' { - return agentName === 'java'; +export function isJavaAgentName(agentName: string | undefined) { + return agentName === agentNames.java; } diff --git a/x-pack/plugins/apm/common/apm_saved_object_constants.ts b/x-pack/plugins/apm/common/apm_saved_object_constants.ts index 0529d90fe940..ac43b700117c 100644 --- a/x-pack/plugins/apm/common/apm_saved_object_constants.ts +++ b/x-pack/plugins/apm/common/apm_saved_object_constants.ts @@ -4,13 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ -// the types have to match the names of the saved object mappings -// in /x-pack/legacy/plugins/apm/mappings.json +// APM Services telemetry +export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE = + 'apm-services-telemetry'; +export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID = 'apm-services-telemetry'; // APM indices export const APM_INDICES_SAVED_OBJECT_TYPE = 'apm-indices'; export const APM_INDICES_SAVED_OBJECT_ID = 'apm-indices'; - -// APM telemetry -export const APM_TELEMETRY_SAVED_OBJECT_TYPE = 'apm-telemetry'; -export const APM_TELEMETRY_SAVED_OBJECT_ID = 'apm-telemetry'; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts index 63fa749cd9f2..1add2427d16a 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -15,10 +15,7 @@ describe('Transaction', () => { const transaction: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: { - version: 'whatever', - version_major: 8 - }, + observer: 'whatever', agent: { name: 'java', version: 'agent version' @@ -66,10 +63,7 @@ describe('Span', () => { const span: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: { - version: 'whatever', - version_major: 8 - }, + observer: 'whatever', agent: { name: 'java', version: 'agent version' @@ -113,10 +107,7 @@ describe('Span', () => { describe('Error', () => { const errorDoc: AllowUnknownProperties = { '@metadata': 'whatever', - observer: { - version: 'whatever', - version_major: 8 - }, + observer: 'whatever', agent: { name: 'java', version: 'agent version' diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index bc1b346f50da..822201baddd8 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -4,24 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ +export const AGENT_NAME = 'agent.name'; export const SERVICE_NAME = 'service.name'; export const SERVICE_ENVIRONMENT = 'service.environment'; export const SERVICE_FRAMEWORK_NAME = 'service.framework.name'; -export const SERVICE_FRAMEWORK_VERSION = 'service.framework.version'; -export const SERVICE_LANGUAGE_NAME = 'service.language.name'; -export const SERVICE_LANGUAGE_VERSION = 'service.language.version'; -export const SERVICE_RUNTIME_NAME = 'service.runtime.name'; -export const SERVICE_RUNTIME_VERSION = 'service.runtime.version'; export const SERVICE_NODE_NAME = 'service.node.name'; export const SERVICE_VERSION = 'service.version'; - -export const AGENT_NAME = 'agent.name'; -export const AGENT_VERSION = 'agent.version'; - export const URL_FULL = 'url.full'; export const HTTP_REQUEST_METHOD = 'http.request.method'; export const USER_ID = 'user.id'; -export const USER_AGENT_ORIGINAL = 'user_agent.original'; export const USER_AGENT_NAME = 'user_agent.name'; export const DESTINATION_ADDRESS = 'destination.address'; diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index dadb1dff6d7a..96579377c95e 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -3,11 +3,8 @@ "server": true, "version": "8.0.0", "kibanaVersion": "kibana", - "configPath": [ - "xpack", - "apm" - ], + "configPath": ["xpack", "apm"], "ui": false, "requiredPlugins": ["apm_oss", "data", "home", "licensing"], - "optionalPlugins": ["cloud", "usageCollection", "taskManager"] + "optionalPlugins": ["cloud", "usageCollection"] } diff --git a/x-pack/plugins/apm/server/index.ts b/x-pack/plugins/apm/server/index.ts index 77655568a7e9..8afdb9e99c1a 100644 --- a/x-pack/plugins/apm/server/index.ts +++ b/x-pack/plugins/apm/server/index.ts @@ -29,8 +29,7 @@ export const config = { enabled: schema.boolean({ defaultValue: true }), transactionGroupBucketSize: schema.number({ defaultValue: 100 }), maxTraceItems: schema.number({ defaultValue: 1000 }) - }), - telemetryCollectionEnabled: schema.boolean({ defaultValue: true }) + }) }) }; @@ -63,8 +62,7 @@ export function mergeConfigs( 'xpack.apm.ui.maxTraceItems': apmConfig.ui.maxTraceItems, 'xpack.apm.ui.transactionGroupBucketSize': apmConfig.ui.transactionGroupBucketSize, - 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern, - 'xpack.apm.telemetryCollectionEnabled': apmConfig.telemetryCollectionEnabled + 'xpack.apm.autocreateApmIndexPattern': apmConfig.autocreateApmIndexPattern }; } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts new file mode 100644 index 000000000000..c45c74a791ae --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedObjectAttributes } from '../../../../../../../src/core/server'; +import { createApmTelementry, storeApmServicesTelemetry } from '../index'; +import { + APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, + APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID +} from '../../../../common/apm_saved_object_constants'; + +describe('apm_telemetry', () => { + describe('createApmTelementry', () => { + it('should create a ApmTelemetry object with boolean flag and frequency map of the given list of AgentNames', () => { + const apmTelemetry = createApmTelementry([ + 'go', + 'nodejs', + 'go', + 'js-base' + ]); + expect(apmTelemetry.has_any_services).toBe(true); + expect(apmTelemetry.services_per_agent).toMatchObject({ + go: 2, + nodejs: 1, + 'js-base': 1 + }); + }); + it('should ignore undefined or unknown AgentName values', () => { + const apmTelemetry = createApmTelementry([ + 'go', + 'nodejs', + 'go', + 'js-base', + 'example-platform' as any, + undefined as any + ]); + expect(apmTelemetry.services_per_agent).toMatchObject({ + go: 2, + nodejs: 1, + 'js-base': 1 + }); + }); + }); + + describe('storeApmServicesTelemetry', () => { + let apmTelemetry: SavedObjectAttributes; + let savedObjectsClient: any; + + beforeEach(() => { + apmTelemetry = { + has_any_services: true, + services_per_agent: { + go: 2, + nodejs: 1, + 'js-base': 1 + } + }; + savedObjectsClient = { create: jest.fn() }; + }); + + it('should call savedObjectsClient create with the given ApmTelemetry object', () => { + storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); + expect(savedObjectsClient.create.mock.calls[0][1]).toBe(apmTelemetry); + }); + + it('should call savedObjectsClient create with the apm-telemetry document type and ID', () => { + storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); + expect(savedObjectsClient.create.mock.calls[0][0]).toBe( + APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE + ); + expect(savedObjectsClient.create.mock.calls[0][2].id).toBe( + APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID + ); + }); + + it('should call savedObjectsClient create with overwrite: true', () => { + storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); + expect(savedObjectsClient.create.mock.calls[0][2].overwrite).toBe(true); + }); + }); +}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts deleted file mode 100644 index 729ccb73d73f..000000000000 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { merge } from 'lodash'; -import { Logger, CallAPIOptions } from 'kibana/server'; -import { IndicesStatsParams, Client } from 'elasticsearch'; -import { - ESSearchRequest, - ESSearchResponse -} from '../../../../typings/elasticsearch'; -import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; -import { tasks } from './tasks'; -import { APMDataTelemetry } from '../types'; - -type TelemetryTaskExecutor = (params: { - indices: ApmIndicesConfig; - search( - params: TSearchRequest - ): Promise>; - indicesStats( - params: IndicesStatsParams, - options?: CallAPIOptions - ): ReturnType; - transportRequest: (params: { - path: string; - method: 'get'; - }) => Promise; -}) => Promise; - -export interface TelemetryTask { - name: string; - executor: TelemetryTaskExecutor; -} - -export type CollectTelemetryParams = Parameters[0] & { - logger: Logger; -}; - -export function collectDataTelemetry({ - search, - indices, - logger, - indicesStats, - transportRequest -}: CollectTelemetryParams) { - return tasks.reduce((prev, task) => { - return prev.then(async data => { - logger.debug(`Executing APM telemetry task ${task.name}`); - try { - const time = process.hrtime(); - const next = await task.executor({ - search, - indices, - indicesStats, - transportRequest - }); - const took = process.hrtime(time); - - return merge({}, data, next, { - tasks: { - [task.name]: { - took: { - ms: Math.round(took[0] * 1000 + took[1] / 1e6) - } - } - } - }); - } catch (err) { - logger.warn(`Failed executing APM telemetry task ${task.name}`); - logger.warn(err); - return data; - } - }); - }, Promise.resolve({} as APMDataTelemetry)); -} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts deleted file mode 100644 index 415076b6ae11..000000000000 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ /dev/null @@ -1,725 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { flatten, merge, sortBy, sum } from 'lodash'; -import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; -import { AGENT_NAMES } from '../../../../common/agent_name'; -import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; -import { - PROCESSOR_EVENT, - SERVICE_NAME, - AGENT_NAME, - AGENT_VERSION, - ERROR_GROUP_ID, - TRANSACTION_NAME, - PARENT_ID, - SERVICE_FRAMEWORK_NAME, - SERVICE_FRAMEWORK_VERSION, - SERVICE_LANGUAGE_NAME, - SERVICE_LANGUAGE_VERSION, - SERVICE_RUNTIME_NAME, - SERVICE_RUNTIME_VERSION, - USER_AGENT_ORIGINAL -} from '../../../../common/elasticsearch_fieldnames'; -import { Span } from '../../../../typings/es_schemas/ui/span'; -import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; -import { TelemetryTask } from '.'; -import { APMTelemetry } from '../types'; - -const TIME_RANGES = ['1d', 'all'] as const; -type TimeRange = typeof TIME_RANGES[number]; - -export const tasks: TelemetryTask[] = [ - { - name: 'processor_events', - executor: async ({ indices, search }) => { - const indicesByProcessorEvent = { - error: indices['apm_oss.errorIndices'], - metric: indices['apm_oss.metricsIndices'], - span: indices['apm_oss.spanIndices'], - transaction: indices['apm_oss.transactionIndices'], - onboarding: indices['apm_oss.onboardingIndices'], - sourcemap: indices['apm_oss.sourcemapIndices'] - }; - - type ProcessorEvent = keyof typeof indicesByProcessorEvent; - - const jobs: Array<{ - processorEvent: ProcessorEvent; - timeRange: TimeRange; - }> = flatten( - (Object.keys( - indicesByProcessorEvent - ) as ProcessorEvent[]).map(processorEvent => - TIME_RANGES.map(timeRange => ({ processorEvent, timeRange })) - ) - ); - - const allData = await jobs.reduce((prevJob, current) => { - return prevJob.then(async data => { - const { processorEvent, timeRange } = current; - - const response = await search({ - index: indicesByProcessorEvent[processorEvent], - body: { - size: 1, - query: { - bool: { - filter: [ - { term: { [PROCESSOR_EVENT]: processorEvent } }, - ...(timeRange !== 'all' - ? [ - { - range: { - '@timestamp': { - gte: `now-${timeRange}` - } - } - } - ] - : []) - ] - } - }, - sort: { - '@timestamp': 'asc' - }, - _source: ['@timestamp'], - track_total_hits: true - } - }); - - const event = response.hits.hits[0]?._source as { - '@timestamp': number; - }; - - return merge({}, data, { - counts: { - [processorEvent]: { - [timeRange]: response.hits.total.value - } - }, - ...(timeRange === 'all' && event - ? { - retainment: { - [processorEvent]: { - ms: - new Date().getTime() - - new Date(event['@timestamp']).getTime() - } - } - } - : {}) - }); - }); - }, Promise.resolve({} as Record> }>)); - - return allData; - } - }, - { - name: 'agent_configuration', - executor: async ({ indices, search }) => { - const agentConfigurationCount = ( - await search({ - index: indices.apmAgentConfigurationIndex, - body: { - size: 0, - track_total_hits: true - } - }) - ).hits.total.value; - - return { - counts: { - agent_configuration: { - all: agentConfigurationCount - } - } - }; - } - }, - { - name: 'services', - executor: async ({ indices, search }) => { - const servicesPerAgent = await AGENT_NAMES.reduce( - (prevJob, agentName) => { - return prevJob.then(async data => { - const response = await search({ - index: [ - indices['apm_oss.errorIndices'], - indices['apm_oss.spanIndices'], - indices['apm_oss.metricsIndices'], - indices['apm_oss.transactionIndices'] - ], - body: { - size: 0, - query: { - bool: { - filter: [ - { - term: { - [AGENT_NAME]: agentName - } - }, - { - range: { - '@timestamp': { - gte: 'now-1d' - } - } - } - ] - } - }, - aggs: { - services: { - cardinality: { - field: SERVICE_NAME - } - } - } - } - }); - - return { - ...data, - [agentName]: response.aggregations?.services.value || 0 - }; - }); - }, - Promise.resolve({} as Record) - ); - - return { - has_any_services: sum(Object.values(servicesPerAgent)) > 0, - services_per_agent: servicesPerAgent - }; - } - }, - { - name: 'versions', - executor: async ({ search, indices }) => { - const response = await search({ - index: [ - indices['apm_oss.transactionIndices'], - indices['apm_oss.spanIndices'], - indices['apm_oss.errorIndices'] - ], - terminateAfter: 1, - body: { - query: { - exists: { - field: 'observer.version' - } - }, - size: 1, - sort: { - '@timestamp': 'desc' - } - } - }); - - const hit = response.hits.hits[0]?._source as Pick< - Transaction | Span | APMError, - 'observer' - >; - - if (!hit || !hit.observer?.version) { - return {}; - } - - const [major, minor, patch] = hit.observer.version - .split('.') - .map(part => Number(part)); - - return { - versions: { - apm_server: { - major, - minor, - patch - } - } - }; - } - }, - { - name: 'groupings', - executor: async ({ search, indices }) => { - const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; - const errorGroupsCount = ( - await search({ - index: indices['apm_oss.errorIndices'], - body: { - size: 0, - query: { - bool: { - filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d] - } - }, - aggs: { - top_service: { - terms: { - field: SERVICE_NAME, - order: { - error_groups: 'desc' - }, - size: 1 - }, - aggs: { - error_groups: { - cardinality: { - field: ERROR_GROUP_ID - } - } - } - } - } - } - }) - ).aggregations?.top_service.buckets[0]?.error_groups.value; - - const transactionGroupsCount = ( - await search({ - index: indices['apm_oss.transactionIndices'], - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [PROCESSOR_EVENT]: 'transaction' } }, - range1d - ] - } - }, - aggs: { - top_service: { - terms: { - field: SERVICE_NAME, - order: { - transaction_groups: 'desc' - }, - size: 1 - }, - aggs: { - transaction_groups: { - cardinality: { - field: TRANSACTION_NAME - } - } - } - } - } - } - }) - ).aggregations?.top_service.buckets[0]?.transaction_groups.value; - - const tracesPerDayCount = ( - await search({ - index: indices['apm_oss.transactionIndices'], - body: { - query: { - bool: { - filter: [ - { term: { [PROCESSOR_EVENT]: 'transaction' } }, - range1d - ], - must_not: { - exists: { field: PARENT_ID } - } - } - }, - track_total_hits: true, - size: 0 - } - }) - ).hits.total.value; - - const servicesCount = ( - await search({ - index: [ - indices['apm_oss.transactionIndices'], - indices['apm_oss.errorIndices'], - indices['apm_oss.metricsIndices'] - ], - body: { - size: 0, - query: { - bool: { - filter: [range1d] - } - }, - aggs: { - service_name: { - cardinality: { - field: SERVICE_NAME - } - } - } - } - }) - ).aggregations?.service_name.value; - - return { - counts: { - max_error_groups_per_service: { - '1d': errorGroupsCount || 0 - }, - max_transaction_groups_per_service: { - '1d': transactionGroupsCount || 0 - }, - traces: { - '1d': tracesPerDayCount || 0 - }, - services: { - '1d': servicesCount || 0 - } - } - }; - } - }, - { - name: 'integrations', - executor: async ({ transportRequest }) => { - const apmJobs = ['*-high_mean_response_time']; - - const response = (await transportRequest({ - method: 'get', - path: `/_ml/anomaly_detectors/${apmJobs.join(',')}` - })) as { data?: { count: number } }; - - return { - integrations: { - ml: { - all_jobs_count: response.data?.count ?? 0 - } - } - }; - } - }, - { - name: 'agents', - executor: async ({ search, indices }) => { - const size = 3; - - const agentData = await AGENT_NAMES.reduce(async (prevJob, agentName) => { - const data = await prevJob; - - const response = await search({ - index: [ - indices['apm_oss.errorIndices'], - indices['apm_oss.metricsIndices'], - indices['apm_oss.transactionIndices'] - ], - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [AGENT_NAME]: agentName } }, - { range: { '@timestamp': { gte: 'now-1d' } } } - ] - } - }, - sort: { - '@timestamp': 'desc' - }, - aggs: { - [AGENT_VERSION]: { - terms: { - field: AGENT_VERSION, - size - } - }, - [SERVICE_FRAMEWORK_NAME]: { - terms: { - field: SERVICE_FRAMEWORK_NAME, - size - }, - aggs: { - [SERVICE_FRAMEWORK_VERSION]: { - terms: { - field: SERVICE_FRAMEWORK_VERSION, - size - } - } - } - }, - [SERVICE_FRAMEWORK_VERSION]: { - terms: { - field: SERVICE_FRAMEWORK_VERSION, - size - } - }, - [SERVICE_LANGUAGE_NAME]: { - terms: { - field: SERVICE_LANGUAGE_NAME, - size - }, - aggs: { - [SERVICE_LANGUAGE_VERSION]: { - terms: { - field: SERVICE_LANGUAGE_VERSION, - size - } - } - } - }, - [SERVICE_LANGUAGE_VERSION]: { - terms: { - field: SERVICE_LANGUAGE_VERSION, - size - } - }, - [SERVICE_RUNTIME_NAME]: { - terms: { - field: SERVICE_RUNTIME_NAME, - size - }, - aggs: { - [SERVICE_RUNTIME_VERSION]: { - terms: { - field: SERVICE_RUNTIME_VERSION, - size - } - } - } - }, - [SERVICE_RUNTIME_VERSION]: { - terms: { - field: SERVICE_RUNTIME_VERSION, - size - } - } - } - } - }); - - const { aggregations } = response; - - if (!aggregations) { - return data; - } - - const toComposite = ( - outerKey: string | number, - innerKey: string | number - ) => `${outerKey}/${innerKey}`; - - return { - ...data, - [agentName]: { - agent: { - version: aggregations[AGENT_VERSION].buckets.map( - bucket => bucket.key as string - ) - }, - service: { - framework: { - name: aggregations[SERVICE_FRAMEWORK_NAME].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - version: aggregations[SERVICE_FRAMEWORK_VERSION].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - composite: sortBy( - flatten( - aggregations[SERVICE_FRAMEWORK_NAME].buckets.map(bucket => - bucket[SERVICE_FRAMEWORK_VERSION].buckets.map( - versionBucket => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key) - }) - ) - ) - ), - 'doc_count' - ) - .reverse() - .slice(0, size) - .map(composite => composite.name) - }, - language: { - name: aggregations[SERVICE_LANGUAGE_NAME].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - version: aggregations[SERVICE_LANGUAGE_VERSION].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - composite: sortBy( - flatten( - aggregations[SERVICE_LANGUAGE_NAME].buckets.map(bucket => - bucket[SERVICE_LANGUAGE_VERSION].buckets.map( - versionBucket => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key) - }) - ) - ) - ), - 'doc_count' - ) - .reverse() - .slice(0, size) - .map(composite => composite.name) - }, - runtime: { - name: aggregations[SERVICE_RUNTIME_NAME].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - version: aggregations[SERVICE_RUNTIME_VERSION].buckets - .map(bucket => bucket.key as string) - .slice(0, size), - composite: sortBy( - flatten( - aggregations[SERVICE_RUNTIME_NAME].buckets.map(bucket => - bucket[SERVICE_RUNTIME_VERSION].buckets.map( - versionBucket => ({ - doc_count: versionBucket.doc_count, - name: toComposite(bucket.key, versionBucket.key) - }) - ) - ) - ), - 'doc_count' - ) - .reverse() - .slice(0, size) - .map(composite => composite.name) - } - } - } - }; - }, Promise.resolve({} as APMTelemetry['agents'])); - - return { - agents: agentData - }; - } - }, - { - name: 'indices_stats', - executor: async ({ indicesStats, indices }) => { - const response = await indicesStats({ - index: [ - indices.apmAgentConfigurationIndex, - indices['apm_oss.errorIndices'], - indices['apm_oss.metricsIndices'], - indices['apm_oss.onboardingIndices'], - indices['apm_oss.sourcemapIndices'], - indices['apm_oss.spanIndices'], - indices['apm_oss.transactionIndices'] - ] - }); - - return { - indices: { - shards: { - total: response._shards.total - }, - all: { - total: { - docs: { - count: response._all.total.docs.count - }, - store: { - size_in_bytes: response._all.total.store.size_in_bytes - } - } - } - } - }; - } - }, - { - name: 'cardinality', - executor: async ({ search }) => { - const allAgentsCardinalityResponse = await search({ - body: { - size: 0, - query: { - bool: { - filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }] - } - }, - aggs: { - [TRANSACTION_NAME]: { - cardinality: { - field: TRANSACTION_NAME - } - }, - [USER_AGENT_ORIGINAL]: { - cardinality: { - field: USER_AGENT_ORIGINAL - } - } - } - } - }); - - const rumAgentCardinalityResponse = await search({ - body: { - size: 0, - query: { - bool: { - filter: [ - { range: { '@timestamp': { gte: 'now-1d' } } }, - { terms: { [AGENT_NAME]: ['rum-js', 'js-base'] } } - ] - } - }, - aggs: { - [TRANSACTION_NAME]: { - cardinality: { - field: TRANSACTION_NAME - } - }, - [USER_AGENT_ORIGINAL]: { - cardinality: { - field: USER_AGENT_ORIGINAL - } - } - } - } - }); - - return { - cardinality: { - transaction: { - name: { - all_agents: { - '1d': - allAgentsCardinalityResponse.aggregations?.[TRANSACTION_NAME] - .value - }, - rum: { - '1d': - rumAgentCardinalityResponse.aggregations?.[TRANSACTION_NAME] - .value - } - } - }, - user_agent: { - original: { - all_agents: { - '1d': - allAgentsCardinalityResponse.aggregations?.[ - USER_AGENT_ORIGINAL - ].value - }, - rum: { - '1d': - rumAgentCardinalityResponse.aggregations?.[ - USER_AGENT_ORIGINAL - ].value - } - } - } - } - }; - } - } -]; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index c80057a2894d..a2b049473082 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -3,127 +3,60 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { CoreSetup, Logger } from 'src/core/server'; -import { Observable } from 'rxjs'; -import { take } from 'rxjs/operators'; -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; + +import { countBy } from 'lodash'; +import { SavedObjectAttributes } from '../../../../../../src/core/server'; +import { isAgentName } from '../../../common/agent_name'; import { - TaskManagerStartContract, - TaskManagerSetupContract -} from '../../../../task_manager/server'; -import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; -import { - APM_TELEMETRY_SAVED_OBJECT_ID, - APM_TELEMETRY_SAVED_OBJECT_TYPE + APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, + APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID } from '../../../common/apm_saved_object_constants'; -import { - collectDataTelemetry, - CollectTelemetryParams -} from './collect_data_telemetry'; -import { APMConfig } from '../..'; -import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; +import { UsageCollectionSetup } from '../../../../../../src/plugins/usage_collection/server'; +import { InternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; -const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; - -export async function createApmTelemetry({ - core, - config$, - usageCollector, - taskManager, - logger -}: { - core: CoreSetup; - config$: Observable; - usageCollector: UsageCollectionSetup; - taskManager: TaskManagerSetupContract; - logger: Logger; -}) { - const savedObjectsClient = await getInternalSavedObjectsClient(core); - - const collectAndStore = async () => { - const config = await config$.pipe(take(1)).toPromise(); - const esClient = core.elasticsearch.dataClient; - - const indices = await getApmIndices({ - config, - savedObjectsClient - }); - - const search = esClient.callAsInternalUser.bind( - esClient, - 'search' - ) as CollectTelemetryParams['search']; - - const indicesStats = esClient.callAsInternalUser.bind( - esClient, - 'indices.stats' - ) as CollectTelemetryParams['indicesStats']; - - const transportRequest = esClient.callAsInternalUser.bind( - esClient, - 'transport.request' - ) as CollectTelemetryParams['transportRequest']; - - const dataTelemetry = await collectDataTelemetry({ - search, - indices, - logger, - indicesStats, - transportRequest - }); - - await savedObjectsClient.create( - APM_TELEMETRY_SAVED_OBJECT_TYPE, - dataTelemetry, - { id: APM_TELEMETRY_SAVED_OBJECT_TYPE, overwrite: true } - ); +export function createApmTelementry( + agentNames: string[] = [] +): SavedObjectAttributes { + const validAgentNames = agentNames.filter(isAgentName); + return { + has_any_services: validAgentNames.length > 0, + services_per_agent: countBy(validAgentNames) }; +} - taskManager.registerTaskDefinitions({ - [APM_TELEMETRY_TASK_NAME]: { - title: 'Collect APM telemetry', - type: APM_TELEMETRY_TASK_NAME, - createTaskRunner: () => { - return { - run: async () => { - await collectAndStore(); - } - }; - } +export async function storeApmServicesTelemetry( + savedObjectsClient: InternalSavedObjectsClient, + apmTelemetry: SavedObjectAttributes +) { + return savedObjectsClient.create( + APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, + apmTelemetry, + { + id: APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID, + overwrite: true } - }); + ); +} - const collector = usageCollector.makeUsageCollector({ +export function makeApmUsageCollector( + usageCollector: UsageCollectionSetup, + savedObjectsRepository: InternalSavedObjectsClient +) { + const apmUsageCollector = usageCollector.makeUsageCollector({ type: 'apm', fetch: async () => { - const data = ( - await savedObjectsClient.get( - APM_TELEMETRY_SAVED_OBJECT_TYPE, - APM_TELEMETRY_SAVED_OBJECT_ID - ) - ).attributes; - - return data; + try { + const apmTelemetrySavedObject = await savedObjectsRepository.get( + APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, + APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID + ); + return apmTelemetrySavedObject.attributes; + } catch (err) { + return createApmTelementry(); + } }, isReady: () => true }); - usageCollector.registerCollector(collector); - - core.getStartServices().then(([coreStart, pluginsStart]) => { - const { taskManager: taskManagerStart } = pluginsStart as { - taskManager: TaskManagerStartContract; - }; - - taskManagerStart.ensureScheduled({ - id: APM_TELEMETRY_TASK_NAME, - taskType: APM_TELEMETRY_TASK_NAME, - schedule: { - interval: '720m' - }, - scope: ['apm'], - params: {}, - state: {} - }); - }); + usageCollector.registerCollector(apmUsageCollector); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts deleted file mode 100644 index f68dc517a222..000000000000 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { DeepPartial } from 'utility-types'; -import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; - -export interface TimeframeMap { - '1d': number; - all: number; -} - -export type TimeframeMap1d = Pick; -export type TimeframeMapAll = Pick; - -export type APMDataTelemetry = DeepPartial<{ - has_any_services: boolean; - services_per_agent: Record; - versions: { - apm_server: { - minor: number; - major: number; - patch: number; - }; - }; - counts: { - transaction: TimeframeMap; - span: TimeframeMap; - error: TimeframeMap; - metric: TimeframeMap; - sourcemap: TimeframeMap; - onboarding: TimeframeMap; - agent_configuration: TimeframeMapAll; - max_transaction_groups_per_service: TimeframeMap; - max_error_groups_per_service: TimeframeMap; - traces: TimeframeMap; - services: TimeframeMap; - }; - cardinality: { - user_agent: { - original: { - all_agents: TimeframeMap1d; - rum: TimeframeMap1d; - }; - }; - transaction: { - name: { - all_agents: TimeframeMap1d; - rum: TimeframeMap1d; - }; - }; - }; - retainment: Record< - 'span' | 'transaction' | 'error' | 'metric' | 'sourcemap' | 'onboarding', - { ms: number } - >; - integrations: { - ml: { - all_jobs_count: number; - }; - }; - agents: Record< - AgentName, - { - agent: { - version: string[]; - }; - service: { - framework: { - name: string[]; - version: string[]; - composite: string[]; - }; - language: { - name: string[]; - version: string[]; - composite: string[]; - }; - runtime: { - name: string[]; - version: string[]; - composite: string[]; - }; - }; - } - >; - indices: { - shards: { - total: number; - }; - all: { - total: { - docs: { - count: number; - }; - store: { - size_in_bytes: number; - }; - }; - }; - }; - tasks: Record< - | 'processor_events' - | 'agent_configuration' - | 'services' - | 'versions' - | 'groupings' - | 'integrations' - | 'agents' - | 'indices_stats' - | 'cardinality', - { took: { ms: number } } - >; -}>; - -export type APMTelemetry = APMDataTelemetry; diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts index 8e8cf698a84c..40a2a0e7216a 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts @@ -39,19 +39,6 @@ function getMockRequest() { _debug: false } }, - __LEGACY: { - server: { - plugins: { - elasticsearch: { - getCluster: jest.fn().mockReturnValue({ callWithInternalUser: {} }) - } - }, - savedObjects: { - SavedObjectsClient: jest.fn(), - getSavedObjectsRepository: jest.fn() - } - } - }, core: { elasticsearch: { dataClient: { diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index a29b9399d843..db14730f802a 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -8,9 +8,9 @@ import { Observable, combineLatest, AsyncSubject } from 'rxjs'; import { map, take } from 'rxjs/operators'; import { Server } from 'hapi'; import { once } from 'lodash'; -import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; -import { TaskManagerSetupContract } from '../../task_manager/server'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { APMOSSPluginSetup } from '../../../../src/plugins/apm_oss/server'; +import { makeApmUsageCollector } from './lib/apm_telemetry'; import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index'; import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index'; import { createApmApi } from './routes/create_apm_api'; @@ -21,7 +21,6 @@ import { tutorialProvider } from './tutorial'; import { CloudSetup } from '../../cloud/server'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; import { LicensingPluginSetup } from '../../licensing/public'; -import { createApmTelemetry } from './lib/apm_telemetry'; export interface LegacySetup { server: Server; @@ -48,10 +47,9 @@ export class APMPlugin implements Plugin { licensing: LicensingPluginSetup; cloud?: CloudSetup; usageCollection?: UsageCollectionSetup; - taskManager?: TaskManagerSetupContract; } ) { - const logger = this.initContext.logger.get(); + const logger = this.initContext.logger.get('apm'); const config$ = this.initContext.config.create(); const mergedConfig$ = combineLatest(plugins.apm_oss.config$, config$).pipe( map(([apmOssConfig, apmConfig]) => mergeConfigs(apmOssConfig, apmConfig)) @@ -63,20 +61,6 @@ export class APMPlugin implements Plugin { const currentConfig = await mergedConfig$.pipe(take(1)).toPromise(); - if ( - plugins.taskManager && - plugins.usageCollection && - currentConfig['xpack.apm.telemetryCollectionEnabled'] - ) { - createApmTelemetry({ - core, - config$: mergedConfig$, - usageCollector: plugins.usageCollection, - taskManager: plugins.taskManager, - logger - }); - } - // create agent configuration index without blocking setup lifecycle createApmAgentConfigurationIndex({ esClient: core.elasticsearch.dataClient, @@ -105,6 +89,18 @@ export class APMPlugin implements Plugin { }) ); + const usageCollection = plugins.usageCollection; + if (usageCollection) { + getInternalSavedObjectsClient(core) + .then(savedObjectsClient => { + makeApmUsageCollector(usageCollection, savedObjectsClient); + }) + .catch(error => { + logger.error('Unable to initialize use collection'); + logger.error(error.message); + }); + } + return { config$: mergedConfig$, registerLegacyAPI: once((__LEGACY: LegacySetup) => { @@ -119,7 +115,6 @@ export class APMPlugin implements Plugin { }; } - public async start() {} - + public start() {} public stop() {} } diff --git a/x-pack/plugins/apm/server/routes/create_api/index.test.ts b/x-pack/plugins/apm/server/routes/create_api/index.test.ts index 312dae1d1f9d..e639bb5101e2 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.test.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.test.ts @@ -36,7 +36,6 @@ const getCoreMock = () => { put, createRouter, context: { - measure: () => undefined, config$: new BehaviorSubject({} as APMConfig), logger: ({ error: jest.fn() diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 1c6561ee24c9..2d4fae9d2707 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -5,6 +5,11 @@ */ import * as t from 'io-ts'; +import { AgentName } from '../../typings/es_schemas/ui/fields/agent'; +import { + createApmTelementry, + storeApmServicesTelemetry +} from '../lib/apm_telemetry'; import { setupRequest } from '../lib/helpers/setup_request'; import { getServiceAgentName } from '../lib/services/get_service_agent_name'; import { getServices } from '../lib/services/get_services'; @@ -13,6 +18,7 @@ import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadat import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceAnnotations } from '../lib/services/annotations'; +import { getInternalSavedObjectsClient } from '../lib/helpers/get_internal_saved_objects_client'; export const servicesRoute = createRoute(core => ({ path: '/api/apm/services', @@ -23,6 +29,16 @@ export const servicesRoute = createRoute(core => ({ const setup = await setupRequest(context, request); const services = await getServices(setup); + // Store telemetry data derived from services + const agentNames = services.items.map( + ({ agentName }) => agentName as AgentName + ); + const apmTelemetry = createApmTelementry(agentNames); + const savedObjectsClient = await getInternalSavedObjectsClient(core); + storeApmServicesTelemetry(savedObjectsClient, apmTelemetry).catch(error => { + context.logger.error(error.message); + }); + return services; } })); diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 8a8d256cf427..6d3620f11a87 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -126,16 +126,6 @@ export interface AggregationOptionsByType { combine_script: Script; reduce_script: Script; }; - date_range: { - field: string; - format?: string; - ranges: Array< - | { from: string | number } - | { to: string | number } - | { from: string | number; to: string | number } - >; - keyed?: boolean; - }; } type AggregationType = keyof AggregationOptionsByType; @@ -146,15 +136,6 @@ type AggregationOptionsMap = Unionize< } > & { aggs?: AggregationInputMap }; -interface DateRangeBucket { - key: string; - to?: number; - from?: number; - to_as_string?: string; - from_as_string?: string; - doc_count: number; -} - export interface AggregationInputMap { [key: string]: AggregationOptionsMap; } @@ -295,11 +276,6 @@ interface AggregationResponsePart< scripted_metric: { value: unknown; }; - date_range: { - buckets: TAggregationOptionsMap extends { date_range: { keyed: true } } - ? Record - : { buckets: DateRangeBucket[] }; - }; } // Type for debugging purposes. If you see an error in AggregationResponseMap @@ -309,7 +285,7 @@ interface AggregationResponsePart< // type MissingAggregationResponseTypes = Exclude< // AggregationType, -// keyof AggregationResponsePart<{}, unknown> +// keyof AggregationResponsePart<{}> // >; export type AggregationResponseMap< diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts index 8e49d02beb90..daf65e44980b 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts @@ -15,7 +15,6 @@ import { Service } from './fields/service'; import { IStackframe } from './fields/stackframe'; import { Url } from './fields/url'; import { User } from './fields/user'; -import { Observer } from './fields/observer'; interface Processor { name: 'error'; @@ -62,5 +61,4 @@ export interface ErrorRaw extends APMBaseDoc { service: Service; url?: Url; user?: User; - observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts b/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts deleted file mode 100644 index 42843130ec47..000000000000 --- a/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export interface Observer { - version: string; - version_major: number; -} diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts index 4d5d2c5c4a12..dbd9e7ede425 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts @@ -6,7 +6,6 @@ import { APMBaseDoc } from './apm_base_doc'; import { IStackframe } from './fields/stackframe'; -import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -51,5 +50,4 @@ export interface SpanRaw extends APMBaseDoc { transaction?: { id: string; }; - observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts index b8ebb4cf8da5..3673f1f13c40 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts @@ -15,7 +15,6 @@ import { Service } from './fields/service'; import { Url } from './fields/url'; import { User } from './fields/user'; import { UserAgent } from './fields/user_agent'; -import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -62,5 +61,4 @@ export interface TransactionRaw extends APMBaseDoc { url?: Url; user?: User; user_agent?: UserAgent; - observer?: Observer; }