From daa81c93065034bb9e34c37cb93485c9009e854c Mon Sep 17 00:00:00 2001 From: Walter Rafelsberger Date: Mon, 19 Jul 2021 12:48:59 +0200 Subject: [PATCH] [ML] Fix custom index name settings, functional tests for APM Latency Correlation. (#105200) Adds first functional tests for APM Latency Correlations code. - Fix: The log log chart's Y axis would only start at 1 by default which hides results for small datasets like the ones used in these tests. Starting the axis at 0 isn't supported to log based ones so we're setting it to just a small value above 0. - Fix: Instead of the hard coded apm-* index we passed on from the client, we now correctly consider APM's custom settings. --- .../search_strategies/correlations/types.ts | 5 +- .../app/correlations/correlations_chart.tsx | 11 +- .../components/app/correlations/index.tsx | 7 +- .../correlations/ml_latency_correlations.tsx | 121 +++++++++++---- .../app/correlations/use_correlations.ts | 4 +- .../service_inventory/service_list/index.tsx | 1 + .../templates/apm_service_template/index.tsx | 4 +- .../correlations/async_search_service.ts | 29 +++- .../correlations/get_query_with_params.ts | 4 +- .../correlations/query_correlation.ts | 6 +- .../correlations/query_field_candidates.ts | 6 +- .../correlations/query_field_value_pairs.ts | 6 +- .../correlations/query_fractions.ts | 6 +- .../correlations/query_histogram.ts | 6 +- .../correlations/query_histogram_interval.ts | 6 +- .../query_histogram_range_steps.ts | 6 +- .../correlations/query_percentiles.ts | 6 +- .../correlations/query_ranges.ts | 6 +- .../correlations/search_strategy.test.ts | 60 ++++++-- .../correlations/search_strategy.ts | 16 +- x-pack/plugins/apm/server/plugin.ts | 25 +++- .../tests/correlations/latency_ml.ts | 23 ++- .../functional/apps/apm/correlations/index.ts | 15 ++ .../apm/correlations/latency_correlations.ts | 139 ++++++++++++++++++ x-pack/test/functional/apps/apm/index.ts | 1 + 25 files changed, 420 insertions(+), 99 deletions(-) create mode 100644 x-pack/test/functional/apps/apm/correlations/index.ts create mode 100644 x-pack/test/functional/apps/apm/correlations/latency_correlations.ts diff --git a/x-pack/plugins/apm/common/search_strategies/correlations/types.ts b/x-pack/plugins/apm/common/search_strategies/correlations/types.ts index 1698708aeb77..6e1fd115aace 100644 --- a/x-pack/plugins/apm/common/search_strategies/correlations/types.ts +++ b/x-pack/plugins/apm/common/search_strategies/correlations/types.ts @@ -19,7 +19,6 @@ export interface ResponseHit { } export interface SearchServiceParams { - index: string; environment?: string; kuery?: string; serviceName?: string; @@ -31,6 +30,10 @@ export interface SearchServiceParams { percentileThresholdValue?: number; } +export interface SearchServiceFetchParams extends SearchServiceParams { + index: string; +} + export interface SearchServiceValue { histogram: HistogramItem[]; value: string; diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx index f4e39c37e289..cfc57d3b3e4a 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx +++ b/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx @@ -70,6 +70,11 @@ const chartTheme: PartialTheme = { }, }; +// Log based axis cannot start a 0. Use a small positive number instead. +const yAxisDomain = { + min: 0.00001, +}; + interface CorrelationsChartProps { field?: string; value?: string; @@ -140,7 +145,10 @@ export function CorrelationsChart({ const histogram = replaceHistogramDotsWithBars(originalHistogram); return ( -
+
{ setIsFlyoutVisible(true); @@ -147,13 +148,17 @@ export function Correlations() { {isFlyoutVisible && ( setIsFlyoutVisible(false)} > -

+

{CORRELATIONS_TITLE}   (enableInspectEsQueries); + const { + ccsWarning, + log, error, histograms, percentileThresholdValue, @@ -76,7 +87,6 @@ export function MlLatencyCorrelations({ onClose }: Props) { cancelFetch, overallHistogram: originalOverallHistogram, } = useCorrelations({ - index: 'apm-*', ...{ ...{ environment, @@ -286,9 +296,10 @@ export function MlLatencyCorrelations({ onClose }: Props) { - + + {ccsWarning && ( + <> + + +

+ {i18n.translate( + 'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutBody', + { + defaultMessage: + 'Data for the correlation analysis could not be fully retrieved. This feature is supported only for 7.14 and later versions.', + } + )} +

+ + + )} {overallHistogram !== undefined ? ( <> -

+

{i18n.translate( 'xpack.apm.correlations.latencyCorrelations.chartTitle', { @@ -341,32 +376,58 @@ export function MlLatencyCorrelations({ onClose }: Props) { ) : null} - {histograms.length > 0 && selectedHistogram !== undefined && ( - +
+ {histograms.length > 0 && selectedHistogram !== undefined && ( + + )} + {histograms.length < 1 && progress > 0.99 ? ( + <> + + + + + + ) : null} +
+ {log.length > 0 && displayLog && ( + + + {log.map((d, i) => { + const splitItem = d.split(': '); + return ( +

+ + {splitItem[0]} {splitItem[1]} + +

+ ); + })} + + )} - {histograms.length < 1 && progress > 0.99 ? ( - <> - - - - - - ) : null} ); } diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts index 2baeb63fa4a2..05cb367a9fde 100644 --- a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts +++ b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts @@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public' import { ApmPluginStartDeps } from '../../../plugin'; interface CorrelationsOptions { - index: string; environment?: string; kuery?: string; serviceName?: string; @@ -37,6 +36,7 @@ interface RawResponse { values: SearchServiceValue[]; overallHistogram: HistogramItem[]; log: string[]; + ccsWarning: boolean; } export const useCorrelations = (params: CorrelationsOptions) => { @@ -106,6 +106,8 @@ export const useCorrelations = (params: CorrelationsOptions) => { }; return { + ccsWarning: rawResponse?.ccsWarning ?? false, + log: rawResponse?.log ?? [], error, histograms: rawResponse?.values ?? [], percentileThresholdValue: diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx index b4644068fd78..a3b0ec0ac66d 100644 --- a/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx +++ b/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx @@ -117,6 +117,7 @@ export function getServiceColumns({ )} - <>{serviceName} +

+ {serviceName} +

diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts index 155cb1f4615b..90d24b6587f4 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts @@ -7,6 +7,7 @@ import { shuffle, range } from 'lodash'; import type { ElasticsearchClient } from 'src/core/server'; +import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; import { fetchTransactionDurationFieldCandidates } from './query_field_candidates'; import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs'; import { fetchTransactionDurationPercentiles } from './query_percentiles'; @@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges'; import type { AsyncSearchProviderProgress, SearchServiceParams, + SearchServiceFetchParams, SearchServiceValue, } from '../../../../common/search_strategies/correlations/types'; import { computeExpectationsAndRanges } from './utils/aggregation_utils'; @@ -28,11 +30,14 @@ const currentTimeAsString = () => new Date().toISOString(); export const asyncSearchServiceProvider = ( esClient: ElasticsearchClient, - params: SearchServiceParams + getApmIndices: () => Promise, + searchServiceParams: SearchServiceParams, + includeFrozen: boolean ) => { let isCancelled = false; let isRunning = true; let error: Error; + let ccsWarning = false; const log: string[] = []; const logMessage = (message: string) => log.push(`${currentTimeAsString()}: ${message}`); @@ -63,7 +68,15 @@ export const asyncSearchServiceProvider = ( }; const fetchCorrelations = async () => { + let params: SearchServiceFetchParams | undefined; + try { + const indices = await getApmIndices(); + params = { + ...searchServiceParams, + index: indices['apm_oss.transactionIndices'], + }; + // 95th percentile to be displayed as a marker in the log log chart const { totalDocs, @@ -172,7 +185,7 @@ export const asyncSearchServiceProvider = ( async function* fetchTransactionDurationHistograms() { for (const item of shuffle(fieldValuePairs)) { - if (item === undefined || isCancelled) { + if (params === undefined || item === undefined || isCancelled) { isRunning = false; return; } @@ -222,10 +235,15 @@ export const asyncSearchServiceProvider = ( yield undefined; } } catch (e) { - // don't fail the whole process for individual correlation queries, just add the error to the internal log. + // don't fail the whole process for individual correlation queries, + // just add the error to the internal log and check if we'd want to set the + // cross-cluster search compatibility warning to true. logMessage( `Failed to fetch correlation/kstest for '${item.field}/${item.value}'` ); + if (params?.index.includes(':')) { + ccsWarning = true; + } yield undefined; } } @@ -247,6 +265,10 @@ export const asyncSearchServiceProvider = ( error = e; } + if (error !== undefined && params?.index.includes(':')) { + ccsWarning = true; + } + isRunning = false; }; @@ -256,6 +278,7 @@ export const asyncSearchServiceProvider = ( const sortedValues = values.sort((a, b) => b.correlation - a.correlation); return { + ccsWarning, error, log, isRunning, diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts index 5d4af3e80f8b..aeb76c37e526 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts @@ -11,7 +11,7 @@ import { pipe } from 'fp-ts/lib/pipeable'; import * as t from 'io-ts'; import { failure } from 'io-ts/lib/PathReporter'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { rangeRt } from '../../../routes/default_api_types'; import { getCorrelationsFilters } from '../../correlations/get_filters'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; @@ -40,7 +40,7 @@ export const getTermsQuery = ( }; interface QueryParams { - params: SearchServiceParams; + params: SearchServiceFetchParams; fieldName?: string; fieldValue?: string; } diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts index f63c36f90d72..94a708f67860 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts @@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; @@ -40,7 +40,7 @@ export interface BucketCorrelation { } export const getTransactionDurationCorrelationRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], @@ -95,7 +95,7 @@ export const getTransactionDurationCorrelationRequest = ( export const fetchTransactionDurationCorrelation = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, expectations: number[], ranges: estypes.AggregationsAggregationRange[], fractions: number[], diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts index 0fbdfef405e0..8aa54e243eec 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts @@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { Field } from './query_field_value_pairs'; @@ -37,7 +37,7 @@ export const hasPrefixToInclude = (fieldName: string) => { }; export const getRandomDocsRequest = ( - params: SearchServiceParams + params: SearchServiceFetchParams ): estypes.SearchRequest => ({ index: params.index, body: { @@ -56,7 +56,7 @@ export const getRandomDocsRequest = ( export const fetchTransactionDurationFieldCandidates = async ( esClient: ElasticsearchClient, - params: SearchServiceParams + params: SearchServiceFetchParams ): Promise<{ fieldCandidates: Field[] }> => { const { index } = params; // Get all fields with keyword mapping diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts index 8fde9d3ab137..23928565da08 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts @@ -11,7 +11,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { AsyncSearchProviderProgress, - SearchServiceParams, + SearchServiceFetchParams, } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; @@ -26,7 +26,7 @@ type FieldValuePairs = FieldValuePair[]; export type Field = string; export const getTermsAggRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, fieldName: string ): estypes.SearchRequest => ({ index: params.index, @@ -46,7 +46,7 @@ export const getTermsAggRequest = ( export const fetchTransactionDurationFieldValuePairs = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, fieldCandidates: Field[], progress: AsyncSearchProviderProgress ): Promise => { diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts index 3d623a4df8c3..e9cec25673c6 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts @@ -7,12 +7,12 @@ import { ElasticsearchClient } from 'kibana/server'; import { estypes } from '@elastic/elasticsearch'; -import { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; export const getTransactionDurationRangesRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, ranges: estypes.AggregationsAggregationRange[] ): estypes.SearchRequest => ({ index: params.index, @@ -35,7 +35,7 @@ export const getTransactionDurationRangesRequest = ( */ export const fetchTransactionDurationFractions = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, ranges: estypes.AggregationsAggregationRange[] ): Promise<{ fractions: number[]; totalDocCount: number }> => { const resp = await esClient.search( diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts index 6f61ecbfdcf0..045caabeab26 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts @@ -13,13 +13,13 @@ import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldname import type { HistogramItem, ResponseHit, - SearchServiceParams, + SearchServiceFetchParams, } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; export const getTransactionDurationHistogramRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, interval: number, fieldName?: string, fieldValue?: string @@ -42,7 +42,7 @@ export const getTransactionDurationHistogramRequest = ( export const fetchTransactionDurationHistogram = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, interval: number, fieldName?: string, fieldValue?: string diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts index c4d1abf24b4d..0f897f2e9236 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts @@ -10,14 +10,14 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; const HISTOGRAM_INTERVALS = 1000; export const getHistogramIntervalRequest = ( - params: SearchServiceParams + params: SearchServiceFetchParams ): estypes.SearchRequest => ({ index: params.index, body: { @@ -32,7 +32,7 @@ export const getHistogramIntervalRequest = ( export const fetchTransactionDurationHistogramInterval = async ( esClient: ElasticsearchClient, - params: SearchServiceParams + params: SearchServiceFetchParams ): Promise => { const resp = await esClient.search(getHistogramIntervalRequest(params)); diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts index 6ee5dd6bcdf8..ba57de2cfde3 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts @@ -19,7 +19,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; @@ -32,7 +32,7 @@ const getHistogramRangeSteps = (min: number, max: number, steps: number) => { }; export const getHistogramIntervalRequest = ( - params: SearchServiceParams + params: SearchServiceFetchParams ): estypes.SearchRequest => ({ index: params.index, body: { @@ -47,7 +47,7 @@ export const getHistogramIntervalRequest = ( export const fetchTransactionDurationHistogramRangeSteps = async ( esClient: ElasticsearchClient, - params: SearchServiceParams + params: SearchServiceFetchParams ): Promise => { const steps = 100; diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts index c80f5d836c0e..cb302f19a000 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts @@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; import { SIGNIFICANT_VALUE_DIGITS } from './constants'; @@ -28,7 +28,7 @@ interface ResponseHit { } export const getTransactionDurationPercentilesRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, percents?: number[], fieldName?: string, fieldValue?: string @@ -58,7 +58,7 @@ export const getTransactionDurationPercentilesRequest = ( export const fetchTransactionDurationPercentiles = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, percents?: number[], fieldName?: string, fieldValue?: string diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts index 9074e7e0809b..0e813a18fdf4 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts @@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch'; import type { ElasticsearchClient } from 'src/core/server'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; -import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types'; +import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types'; import { getQueryWithParams } from './get_query_with_params'; @@ -27,7 +27,7 @@ interface ResponseHit { } export const getTransactionDurationRangesRequest = ( - params: SearchServiceParams, + params: SearchServiceFetchParams, rangesSteps: number[], fieldName?: string, fieldValue?: string @@ -65,7 +65,7 @@ export const getTransactionDurationRangesRequest = ( export const fetchTransactionDurationRanges = async ( esClient: ElasticsearchClient, - params: SearchServiceParams, + params: SearchServiceFetchParams, rangesSteps: number[], fieldName?: string, fieldValue?: string diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts index 09775cb2eb03..401cda97afef 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts @@ -9,6 +9,8 @@ import type { estypes } from '@elastic/elasticsearch'; import { SearchStrategyDependencies } from 'src/plugins/data/server'; +import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; + import { apmCorrelationsSearchStrategyProvider, PartialSearchRequest, @@ -94,10 +96,19 @@ const clientSearchMock = ( }; }; +const getApmIndicesMock = async () => + ({ + // eslint-disable-next-line @typescript-eslint/naming-convention + 'apm_oss.transactionIndices': 'apm-*', + } as ApmIndicesConfig); + describe('APM Correlations search strategy', () => { describe('strategy interface', () => { it('returns a custom search strategy with a `search` and `cancel` function', async () => { - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + getApmIndicesMock, + false + ); expect(typeof searchStrategy.search).toBe('function'); expect(typeof searchStrategy.cancel).toBe('function'); }); @@ -106,12 +117,14 @@ describe('APM Correlations search strategy', () => { describe('search', () => { let mockClientFieldCaps: jest.Mock; let mockClientSearch: jest.Mock; + let mockGetApmIndicesMock: jest.Mock; let mockDeps: SearchStrategyDependencies; let params: Required['params']; beforeEach(() => { mockClientFieldCaps = jest.fn(clientFieldCapsMock); mockClientSearch = jest.fn(clientSearchMock); + mockGetApmIndicesMock = jest.fn(getApmIndicesMock); mockDeps = ({ esClient: { asCurrentUser: { @@ -121,7 +134,6 @@ describe('APM Correlations search strategy', () => { }, } as unknown) as SearchStrategyDependencies; params = { - index: 'apm-*', start: '2020', end: '2021', }; @@ -130,7 +142,13 @@ describe('APM Correlations search strategy', () => { describe('async functionality', () => { describe('when no params are provided', () => { it('throws an error', async () => { - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); + + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(0); + expect(() => searchStrategy.search({}, {}, mockDeps)).toThrow( 'Invalid request parameters.' ); @@ -139,8 +157,14 @@ describe('APM Correlations search strategy', () => { describe('when no ID is provided', () => { it('performs a client search with params', async () => { - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); await searchStrategy.search({ params }, {}, mockDeps).toPromise(); + + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); + const [[request]] = mockClientSearch.mock.calls; expect(request.index).toEqual('apm-*'); @@ -179,7 +203,10 @@ describe('APM Correlations search strategy', () => { describe('when an ID with params is provided', () => { it('retrieves the current request', async () => { - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); const response = await searchStrategy .search({ params }, {}, mockDeps) .toPromise(); @@ -190,6 +217,7 @@ describe('APM Correlations search strategy', () => { .search({ id: searchStrategyId, params }, {}, mockDeps) .toPromise(); + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); expect(response2).toEqual( expect.objectContaining({ id: searchStrategyId }) ); @@ -201,11 +229,16 @@ describe('APM Correlations search strategy', () => { mockClientSearch .mockReset() .mockRejectedValueOnce(new Error('client error')); - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); const response = await searchStrategy .search({ params }, {}, mockDeps) .toPromise(); + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); + expect(response).toEqual( expect.objectContaining({ isRunning: true }) ); @@ -213,11 +246,15 @@ describe('APM Correlations search strategy', () => { }); it('triggers the subscription only once', async () => { - expect.assertions(1); - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + expect.assertions(2); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); searchStrategy .search({ params }, {}, mockDeps) .subscribe((response) => { + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); expect(response).toEqual( expect.objectContaining({ loaded: 0, isRunning: true }) ); @@ -227,12 +264,16 @@ describe('APM Correlations search strategy', () => { describe('response', () => { it('sends an updated response on consecutive search calls', async () => { - const searchStrategy = await apmCorrelationsSearchStrategyProvider(); + const searchStrategy = await apmCorrelationsSearchStrategyProvider( + mockGetApmIndicesMock, + false + ); const response1 = await searchStrategy .search({ params }, {}, mockDeps) .toPromise(); + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); expect(typeof response1.id).toEqual('string'); expect(response1).toEqual( expect.objectContaining({ loaded: 0, isRunning: true }) @@ -244,6 +285,7 @@ describe('APM Correlations search strategy', () => { .search({ id: response1.id, params }, {}, mockDeps) .toPromise(); + expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1); expect(response2.id).toEqual(response1.id); expect(response2).toEqual( expect.objectContaining({ loaded: 100, isRunning: false }) diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts index 8f2e6913c0d0..3601f19ad705 100644 --- a/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts +++ b/x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts @@ -19,6 +19,8 @@ import type { SearchServiceValue, } from '../../../../common/search_strategies/correlations/types'; +import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; + import { asyncSearchServiceProvider } from './async_search_service'; export type PartialSearchRequest = IKibanaSearchRequest; @@ -26,10 +28,10 @@ export type PartialSearchResponse = IKibanaSearchResponse<{ values: SearchServiceValue[]; }>; -export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< - PartialSearchRequest, - PartialSearchResponse -> => { +export const apmCorrelationsSearchStrategyProvider = ( + getApmIndices: () => Promise, + includeFrozen: boolean +): ISearchStrategy => { const asyncSearchServiceMap = new Map< string, ReturnType @@ -65,7 +67,9 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< } else { getAsyncSearchServiceState = asyncSearchServiceProvider( deps.esClient.asCurrentUser, - request.params + getApmIndices, + request.params, + includeFrozen ); } @@ -73,6 +77,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< const id = request.id ?? uuid(); const { + ccsWarning, error, log, isRunning, @@ -102,6 +107,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy< isRunning, isPartial: isRunning, rawResponse: { + ccsWarning, log, took, values, diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 3a7eb738dd3b..d28e43d9cb97 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -16,6 +16,7 @@ import { PluginInitializerContext, } from 'src/core/server'; import { isEmpty, mapValues, once } from 'lodash'; +import { SavedObjectsClient } from '../../../../src/core/server'; import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets'; import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map'; import { APMConfig, APMXPackConfig, APM_SERVER_FEATURE_ID } from '.'; @@ -248,12 +249,24 @@ export class APMPlugin }); // search strategies for async partial search results - if (plugins.data?.search?.registerSearchStrategy !== undefined) { - plugins.data.search.registerSearchStrategy( - 'apmCorrelationsSearchStrategy', - apmCorrelationsSearchStrategyProvider() - ); - } + core.getStartServices().then(([coreStart]) => { + (async () => { + const savedObjectsClient = new SavedObjectsClient( + coreStart.savedObjects.createInternalRepository() + ); + + plugins.data.search.registerSearchStrategy( + 'apmCorrelationsSearchStrategy', + apmCorrelationsSearchStrategyProvider( + boundGetApmIndices, + await coreStart.uiSettings + .asScopedToClient(savedObjectsClient) + .get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN) + ) + ); + })(); + }); + return { config$: mergedConfig$, getApmIndices: boundGetApmIndices, diff --git a/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts index cc8f48fb5894..bbb2097f6301 100644 --- a/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts +++ b/x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts @@ -26,7 +26,6 @@ export default function ApiTest({ getService }: FtrProviderContext) { const getRequestBody = () => { const partialSearchRequest: PartialSearchRequest = { params: { - index: 'apm-*', environment: 'ENVIRONMENT_ALL', start: '2020', end: '2021', @@ -141,7 +140,7 @@ export default function ApiTest({ getService }: FtrProviderContext) { registry.when( 'Correlations latency_ml with data and opbeans-node args', - { config: 'trial', archives: ['ml_8.0.0'] }, + { config: 'trial', archives: ['8.0.0'] }, () => { // putting this into a single `it` because the responses depend on each other it('queries the search strategy and returns results', async () => { @@ -235,30 +234,30 @@ export default function ApiTest({ getService }: FtrProviderContext) { const { rawResponse: finalRawResponse } = followUpResult; expect(typeof finalRawResponse?.took).to.be('number'); - expect(finalRawResponse?.percentileThresholdValue).to.be(1404927.875); + expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875); expect(finalRawResponse?.overallHistogram.length).to.be(101); expect(finalRawResponse?.values.length).to.eql( - 1, - `Expected 1 identified correlations, got ${finalRawResponse?.values.length}.` + 13, + `Expected 13 identified correlations, got ${finalRawResponse?.values.length}.` ); expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([ - 'Fetched 95th percentile value of 1404927.875 based on 989 documents.', + 'Fetched 95th percentile value of 1309695.875 based on 1244 documents.', 'Loaded histogram range steps.', 'Loaded overall histogram chart data.', 'Loaded percentiles.', - 'Identified 67 fieldCandidates.', - 'Identified 339 fieldValuePairs.', - 'Loaded fractions and totalDocCount of 989.', - 'Identified 1 significant correlations out of 339 field/value pairs.', + 'Identified 69 fieldCandidates.', + 'Identified 379 fieldValuePairs.', + 'Loaded fractions and totalDocCount of 1244.', + 'Identified 13 significant correlations out of 379 field/value pairs.', ]); const correlation = finalRawResponse?.values[0]; expect(typeof correlation).to.be('object'); expect(correlation?.field).to.be('transaction.result'); expect(correlation?.value).to.be('success'); - expect(correlation?.correlation).to.be(0.37418510688551887); - expect(correlation?.ksTest).to.be(1.1238496968312214e-10); + expect(correlation?.correlation).to.be(0.6275246559191225); + expect(correlation?.ksTest).to.be(4.806503252860024e-13); expect(correlation?.histogram.length).to.be(101); }); } diff --git a/x-pack/test/functional/apps/apm/correlations/index.ts b/x-pack/test/functional/apps/apm/correlations/index.ts new file mode 100644 index 000000000000..ae5f594e5440 --- /dev/null +++ b/x-pack/test/functional/apps/apm/correlations/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ loadTestFile }: FtrProviderContext) { + describe('correlations', function () { + this.tags('skipFirefox'); + loadTestFile(require.resolve('./latency_correlations')); + }); +} diff --git a/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts b/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts new file mode 100644 index 000000000000..bc06b7299363 --- /dev/null +++ b/x-pack/test/functional/apps/apm/correlations/latency_correlations.ts @@ -0,0 +1,139 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +export default function ({ getPageObjects, getService }: FtrProviderContext) { + const esArchiver = getService('esArchiver'); + const find = getService('find'); + const retry = getService('retry'); + const spacesService = getService('spaces'); + const PageObjects = getPageObjects(['common', 'error', 'timePicker', 'security']); + const testSubjects = getService('testSubjects'); + const appsMenu = getService('appsMenu'); + + const testData = { serviceName: 'opbeans-go' }; + + describe('latency correlations', () => { + describe('space with no features disabled', () => { + before(async () => { + await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm'); + await spacesService.create({ + id: 'custom_space', + name: 'custom_space', + disabledFeatures: [], + }); + }); + + after(async () => { + await spacesService.delete('custom_space'); + }); + + it('shows apm navlink', async () => { + await PageObjects.common.navigateToApp('home', { + basePath: '/s/custom_space', + }); + const navLinks = (await appsMenu.readLinks()).map((link) => link.text); + expect(navLinks).to.contain('APM'); + }); + + it('can navigate to APM app', async () => { + await PageObjects.common.navigateToApp('apm'); + + await retry.try(async () => { + await testSubjects.existOrFail('apmMainContainer', { + timeout: 10000, + }); + + const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer'); + const apmMainContainerTextItems = apmMainContainerText[0].split('\n'); + expect(apmMainContainerTextItems).to.contain('No services found'); + }); + }); + + it('sets the timePicker to return data', async () => { + await PageObjects.timePicker.timePickerExists(); + + const fromTime = 'Jul 29, 2019 @ 00:00:00.000'; + const toTime = 'Jul 30, 2019 @ 00:00:00.000'; + await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime); + + await retry.try(async () => { + const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer'); + const apmMainContainerTextItems = apmMainContainerText[0].split('\n'); + + expect(apmMainContainerTextItems).to.not.contain('No services found'); + + expect(apmMainContainerTextItems).to.contain('opbeans-go'); + expect(apmMainContainerTextItems).to.contain('opbeans-node'); + expect(apmMainContainerTextItems).to.contain('opbeans-ruby'); + expect(apmMainContainerTextItems).to.contain('opbeans-python'); + expect(apmMainContainerTextItems).to.contain('opbeans-dotnet'); + expect(apmMainContainerTextItems).to.contain('opbeans-java'); + + expect(apmMainContainerTextItems).to.contain('development'); + + const items = await testSubjects.findAll('apmServiceListAppLink'); + expect(items.length).to.be(6); + }); + }); + + it(`navigates to the 'opbeans-go' service overview page`, async function () { + await find.clickByDisplayedLinkText(testData.serviceName); + + await retry.try(async () => { + const apmMainTemplateHeaderServiceName = await testSubjects.getVisibleTextAll( + 'apmMainTemplateHeaderServiceName' + ); + expect(apmMainTemplateHeaderServiceName).to.contain('opbeans-go'); + }); + }); + + it('shows the correlations flyout', async function () { + await testSubjects.click('apmViewCorrelationsButton'); + + await retry.try(async () => { + await testSubjects.existOrFail('apmCorrelationsFlyout', { + timeout: 10000, + }); + + const apmCorrelationsFlyoutHeader = await testSubjects.getVisibleText( + 'apmCorrelationsFlyoutHeader' + ); + + expect(apmCorrelationsFlyoutHeader).to.contain('Correlations BETA'); + }); + }); + + it('loads the correlation results', async function () { + await retry.try(async () => { + // Assert that the data fully loaded to 100% + const apmCorrelationsLatencyCorrelationsProgressTitle = await testSubjects.getVisibleText( + 'apmCorrelationsLatencyCorrelationsProgressTitle' + ); + expect(apmCorrelationsLatencyCorrelationsProgressTitle).to.be('Progress: 100%'); + + // Assert that the Correlations Chart and its header are present + const apmCorrelationsLatencyCorrelationsChartTitle = await testSubjects.getVisibleText( + 'apmCorrelationsLatencyCorrelationsChartTitle' + ); + expect(apmCorrelationsLatencyCorrelationsChartTitle).to.be( + `Latency distribution for ${testData.serviceName}` + ); + await testSubjects.existOrFail('apmCorrelationsChart', { + timeout: 10000, + }); + + // Assert that results for the given service didn't find any correlations + const apmCorrelationsTable = await testSubjects.getVisibleText('apmCorrelationsTable'); + expect(apmCorrelationsTable).to.be('No significant correlations found'); + }); + }); + }); + }); +} diff --git a/x-pack/test/functional/apps/apm/index.ts b/x-pack/test/functional/apps/apm/index.ts index d2531b72e1b5..e4db5a66aa55 100644 --- a/x-pack/test/functional/apps/apm/index.ts +++ b/x-pack/test/functional/apps/apm/index.ts @@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) { describe('APM specs', function () { this.tags('ciGroup6'); loadTestFile(require.resolve('./feature_controls')); + loadTestFile(require.resolve('./correlations')); }); }