maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Fix custom index name settings, functional tests for APM Latency Correlation. (#105200)

Browse source 
Adds first functional tests for APM Latency Correlations code.
- Fix: The log log chart's Y axis would only start at 1 by default which hides results for small datasets like the ones used in these tests. Starting the axis at 0 isn't supported to log based ones so we're setting it to just a small value above 0.
- Fix: Instead of the hard coded apm-* index we passed on from the client, we now correctly consider APM's custom settings.
This commit is contained in:
Walter Rafelsberger 2021-07-19 12:48:59 +02:00 committed by GitHub
parent 15a613f488
commit daa81c9306
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 420 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
x-pack/plugins/apm/common/search_strategies/correlations/types.ts
View file

@ -19,7 +19,6 @@ export interface ResponseHit {
}
export interface SearchServiceParams {
index: string;
environment?: string;
kuery?: string;
serviceName?: string;
@ -31,6 +30,10 @@ export interface SearchServiceParams {
percentileThresholdValue?: number;
}
export interface SearchServiceFetchParams extends SearchServiceParams {
index: string;
}
export interface SearchServiceValue {
histogram: HistogramItem[];
value: string;

11
x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx
View file

@ -70,6 +70,11 @@ const chartTheme: PartialTheme = {
},
};
// Log based axis cannot start a 0. Use a small positive number instead.
const yAxisDomain = {
min: 0.00001,
};
interface CorrelationsChartProps {
field?: string;
value?: string;
@ -140,7 +145,10 @@ export function CorrelationsChart({
const histogram = replaceHistogramDotsWithBars(originalHistogram);
return (
<div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
<div
data-test-subj="apmCorrelationsChart"
style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}
>
<Chart
size={{
height: '250px',
@ -168,6 +176,7 @@ export function CorrelationsChart({
/>
<Axis
id="y-axis"
domain={yAxisDomain}
title={i18n.translate(
'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
{ defaultMessage: '# transactions' }

7
x-pack/plugins/apm/public/components/app/correlations/index.tsx
View file

@ -134,6 +134,7 @@ export function Correlations() {
return (
<>
<EuiButton
data-test-subj="apmViewCorrelationsButton"
fill
onClick={() => {
setIsFlyoutVisible(true);
@ -147,13 +148,17 @@ export function Correlations() {
{isFlyoutVisible && (
<EuiPortal>
<EuiFlyout
data-test-subj="apmCorrelationsFlyout"
size="l"
ownFocus
onClose={() => setIsFlyoutVisible(false)}
>
<EuiFlyoutHeader hasBorder aria-labelledby="correlations-flyout">
<EuiTitle>
<h2 id="correlations-flyout">
<h2
data-test-subj="apmCorrelationsFlyoutHeader"
id="correlations-flyout"
>
{CORRELATIONS_TITLE}
&nbsp;
<EuiBetaBadge

121
x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
View file

@ -8,6 +8,10 @@
import React, { useEffect, useMemo, useState } from 'react';
import { useHistory } from 'react-router-dom';
import {
EuiCallOut,
EuiCode,
EuiAccordion,
EuiPanel,
EuiIcon,
EuiBasicTableColumn,
EuiButton,
@ -34,7 +38,10 @@ import {
} from './correlations_table';
import { useCorrelations } from './use_correlations';
import { push } from '../../shared/Links/url_helpers';
import { useUiTracker } from '../../../../../observability/public';
import {
enableInspectEsQueries,
useUiTracker,
} from '../../../../../observability/public';
import { asPreciseDecimal } from '../../../../common/utils/formatters';
import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context';
import { LatencyCorrelationsHelpPopover } from './ml_latency_correlations_help_popover';
@ -58,7 +65,7 @@ interface MlCorrelationsTerms {
export function MlLatencyCorrelations({ onClose }: Props) {
const {
core: { notifications },
core: { notifications, uiSettings },
} = useApmPluginContext();
const { serviceName, transactionType } = useApmServiceContext();
@ -66,7 +73,11 @@ export function MlLatencyCorrelations({ onClose }: Props) {
const { environment, kuery, transactionName, start, end } = urlParams;
const displayLog = uiSettings.get<boolean>(enableInspectEsQueries);
const {
ccsWarning,
log,
error,
histograms,
percentileThresholdValue,
@ -76,7 +87,6 @@ export function MlLatencyCorrelations({ onClose }: Props) {
cancelFetch,
overallHistogram: originalOverallHistogram,
} = useCorrelations({
index: 'apm-*',
...{
...{
environment,
@ -286,9 +296,10 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</EuiFlexItem>
<EuiFlexItem>
<EuiFlexGroup direction="column" gutterSize="none">
<EuiFlexItem>
<EuiFlexItem data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle">
<EuiText size="xs" color="subdued">
<FormattedMessage
data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle"
id="xpack.apm.correlations.latencyCorrelations.progressTitle"
defaultMessage="Progress: {progress}%"
values={{ progress: Math.round(progress * 100) }}
@ -313,11 +324,35 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</EuiFlexItem>
</EuiFlexGroup>
{ccsWarning && (
<>
<EuiSpacer size="m" />
<EuiCallOut
title={i18n.translate(
'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutTitle',
{
defaultMessage: 'Cross-cluster search compatibility',
}
)}
color="warning"
>
<p>
{i18n.translate(
'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutBody',
{
defaultMessage:
'Data for the correlation analysis could not be fully retrieved. This feature is supported only for 7.14 and later versions.',
}
)}
</p>
</EuiCallOut>
</>
)}
<EuiSpacer size="m" />
{overallHistogram !== undefined ? (
<>
<EuiTitle size="xxs">
<h4>
<h4 data-test-subj="apmCorrelationsLatencyCorrelationsChartTitle">
{i18n.translate(
'xpack.apm.correlations.latencyCorrelations.chartTitle',
{
@ -341,32 +376,58 @@ export function MlLatencyCorrelations({ onClose }: Props) {
</>
) : null}
{histograms.length > 0 && selectedHistogram !== undefined && (
<CorrelationsTable
// @ts-ignore correlations don't have the same column format other tables have
columns={mlCorrelationColumns}
// @ts-expect-error correlations don't have the same significant term other tables have
significantTerms={histogramTerms}
status={FETCH_STATUS.SUCCESS}
setSelectedSignificantTerm={setSelectedSignificantTerm}
selectedTerm={{
fieldName: selectedHistogram.field,
fieldValue: selectedHistogram.value,
}}
onFilter={onClose}
/>
<div data-test-subj="apmCorrelationsTable">
{histograms.length > 0 && selectedHistogram !== undefined && (
<CorrelationsTable
// @ts-ignore correlations don't have the same column format other tables have
columns={mlCorrelationColumns}
// @ts-expect-error correlations don't have the same significant term other tables have
significantTerms={histogramTerms}
status={FETCH_STATUS.SUCCESS}
setSelectedSignificantTerm={setSelectedSignificantTerm}
selectedTerm={{
fieldName: selectedHistogram.field,
fieldValue: selectedHistogram.value,
}}
onFilter={onClose}
/>
)}
{histograms.length < 1 && progress > 0.99 ? (
<>
<EuiSpacer size="m" />
<EuiText textAlign="center">
<FormattedMessage
id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
defaultMessage="No significant correlations found"
/>
</EuiText>
</>
) : null}
</div>
{log.length > 0 && displayLog && (
<EuiAccordion
id="accordion1"
buttonContent={i18n.translate(
'xpack.apm.correlations.latencyCorrelations.logButtonContent',
{
defaultMessage: 'Log',
}
)}
>
<EuiPanel color="subdued">
{log.map((d, i) => {
const splitItem = d.split(': ');
return (
<p key={i}>
<small>
<EuiCode>{splitItem[0]}</EuiCode> {splitItem[1]}
</small>
</p>
);
})}
</EuiPanel>
</EuiAccordion>
)}
{histograms.length < 1 && progress > 0.99 ? (
<>
<EuiSpacer size="m" />
<EuiText textAlign="center">
<FormattedMessage
id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
defaultMessage="No significant correlations found"
/>
</EuiText>
</>
) : null}
</>
);
}

4
x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
View file

@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'
import { ApmPluginStartDeps } from '../../../plugin';
interface CorrelationsOptions {
index: string;
environment?: string;
kuery?: string;
serviceName?: string;
@ -37,6 +36,7 @@ interface RawResponse {
values: SearchServiceValue[];
overallHistogram: HistogramItem[];
log: string[];
ccsWarning: boolean;
}
export const useCorrelations = (params: CorrelationsOptions) => {
@ -106,6 +106,8 @@ export const useCorrelations = (params: CorrelationsOptions) => {
};
return {
ccsWarning: rawResponse?.ccsWarning ?? false,
log: rawResponse?.log ?? [],
error,
histograms: rawResponse?.values ?? [],
percentileThresholdValue:

1
x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx
View file

@ -117,6 +117,7 @@ export function getServiceColumns({
)}
<EuiFlexItem className="apmServiceList__serviceNameContainer">
<AppLink
data-test-subj="apmServiceListAppLink"
serviceName={serviceName}
transactionType={transactionType}
className="eui-textTruncate"

4
x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx
View file

@ -93,7 +93,9 @@ function TemplateWithContext({
<EuiFlexGroup alignItems="center">
<EuiFlexItem grow={false}>
<EuiTitle size="l">
<>{serviceName}</>
<h1 data-test-subj="apmMainTemplateHeaderServiceName">
{serviceName}
</h1>
</EuiTitle>
</EuiFlexItem>
<EuiFlexItem grow={false}>

29
x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
View file

@ -7,6 +7,7 @@
import { shuffle, range } from 'lodash';
import type { ElasticsearchClient } from 'src/core/server';
import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
import { fetchTransactionDurationPercentiles } from './query_percentiles';
@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
import type {
AsyncSearchProviderProgress,
SearchServiceParams,
SearchServiceFetchParams,
SearchServiceValue,
} from '../../../../common/search_strategies/correlations/types';
import { computeExpectationsAndRanges } from './utils/aggregation_utils';
@ -28,11 +30,14 @@ const currentTimeAsString = () => new Date().toISOString();
export const asyncSearchServiceProvider = (
esClient: ElasticsearchClient,
params: SearchServiceParams
getApmIndices: () => Promise<ApmIndicesConfig>,
searchServiceParams: SearchServiceParams,
includeFrozen: boolean
) => {
let isCancelled = false;
let isRunning = true;
let error: Error;
let ccsWarning = false;
const log: string[] = [];
const logMessage = (message: string) =>
log.push(`${currentTimeAsString()}: ${message}`);
@ -63,7 +68,15 @@ export const asyncSearchServiceProvider = (
};
const fetchCorrelations = async () => {
let params: SearchServiceFetchParams | undefined;
try {
const indices = await getApmIndices();
params = {
...searchServiceParams,
index: indices['apm_oss.transactionIndices'],
};
// 95th percentile to be displayed as a marker in the log log chart
const {
totalDocs,
@ -172,7 +185,7 @@ export const asyncSearchServiceProvider = (
async function* fetchTransactionDurationHistograms() {
for (const item of shuffle(fieldValuePairs)) {
if (item === undefined || isCancelled) {
if (params === undefined || item === undefined || isCancelled) {
isRunning = false;
return;
}
@ -222,10 +235,15 @@ export const asyncSearchServiceProvider = (
yield undefined;
}
} catch (e) {
// don't fail the whole process for individual correlation queries, just add the error to the internal log.
// don't fail the whole process for individual correlation queries,
// just add the error to the internal log and check if we'd want to set the
// cross-cluster search compatibility warning to true.
logMessage(
`Failed to fetch correlation/kstest for '${item.field}/${item.value}'`
);
if (params?.index.includes(':')) {
ccsWarning = true;
}
yield undefined;
}
}
@ -247,6 +265,10 @@ export const asyncSearchServiceProvider = (
error = e;
}
if (error !== undefined && params?.index.includes(':')) {
ccsWarning = true;
}
isRunning = false;
};
@ -256,6 +278,7 @@ export const asyncSearchServiceProvider = (
const sortedValues = values.sort((a, b) => b.correlation - a.correlation);
return {
ccsWarning,
error,
log,
isRunning,

4
x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts
View file

@ -11,7 +11,7 @@ import { pipe } from 'fp-ts/lib/pipeable';
import * as t from 'io-ts';
import { failure } from 'io-ts/lib/PathReporter';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { rangeRt } from '../../../routes/default_api_types';
import { getCorrelationsFilters } from '../../correlations/get_filters';
import { Setup, SetupTimeRange } from '../../helpers/setup_request';
@ -40,7 +40,7 @@ export const getTermsQuery = (
};
interface QueryParams {
params: SearchServiceParams;
params: SearchServiceFetchParams;
fieldName?: string;
fieldValue?: string;
}

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts
View file

@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
@ -40,7 +40,7 @@ export interface BucketCorrelation {
}
export const getTransactionDurationCorrelationRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
expectations: number[],
ranges: estypes.AggregationsAggregationRange[],
fractions: number[],
@ -95,7 +95,7 @@ export const getTransactionDurationCorrelationRequest = (
export const fetchTransactionDurationCorrelation = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
expectations: number[],
ranges: estypes.AggregationsAggregationRange[],
fractions: number[],

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts
View file

@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
import { Field } from './query_field_value_pairs';
@ -37,7 +37,7 @@ export const hasPrefixToInclude = (fieldName: string) => {
};
export const getRandomDocsRequest = (
params: SearchServiceParams
params: SearchServiceFetchParams
): estypes.SearchRequest => ({
index: params.index,
body: {
@ -56,7 +56,7 @@ export const getRandomDocsRequest = (
export const fetchTransactionDurationFieldCandidates = async (
esClient: ElasticsearchClient,
params: SearchServiceParams
params: SearchServiceFetchParams
): Promise<{ fieldCandidates: Field[] }> => {
const { index } = params;
// Get all fields with keyword mapping

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts
View file

@ -11,7 +11,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type {
AsyncSearchProviderProgress,
SearchServiceParams,
SearchServiceFetchParams,
} from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
@ -26,7 +26,7 @@ type FieldValuePairs = FieldValuePair[];
export type Field = string;
export const getTermsAggRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
fieldName: string
): estypes.SearchRequest => ({
index: params.index,
@ -46,7 +46,7 @@ export const getTermsAggRequest = (
export const fetchTransactionDurationFieldValuePairs = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
fieldCandidates: Field[],
progress: AsyncSearchProviderProgress
): Promise<FieldValuePairs> => {

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts
View file

@ -7,12 +7,12 @@
import { ElasticsearchClient } from 'kibana/server';
import { estypes } from '@elastic/elasticsearch';
import { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
export const getTransactionDurationRangesRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
ranges: estypes.AggregationsAggregationRange[]
): estypes.SearchRequest => ({
index: params.index,
@ -35,7 +35,7 @@ export const getTransactionDurationRangesRequest = (
*/
export const fetchTransactionDurationFractions = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
ranges: estypes.AggregationsAggregationRange[]
): Promise<{ fractions: number[]; totalDocCount: number }> => {
const resp = await esClient.search(

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts
View file

@ -13,13 +13,13 @@ import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldname
import type {
HistogramItem,
ResponseHit,
SearchServiceParams,
SearchServiceFetchParams,
} from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
export const getTransactionDurationHistogramRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
interval: number,
fieldName?: string,
fieldValue?: string
@ -42,7 +42,7 @@ export const getTransactionDurationHistogramRequest = (
export const fetchTransactionDurationHistogram = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
interval: number,
fieldName?: string,
fieldValue?: string

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts
View file

@ -10,14 +10,14 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
const HISTOGRAM_INTERVALS = 1000;
export const getHistogramIntervalRequest = (
params: SearchServiceParams
params: SearchServiceFetchParams
): estypes.SearchRequest => ({
index: params.index,
body: {
@ -32,7 +32,7 @@ export const getHistogramIntervalRequest = (
export const fetchTransactionDurationHistogramInterval = async (
esClient: ElasticsearchClient,
params: SearchServiceParams
params: SearchServiceFetchParams
): Promise<number> => {
const resp = await esClient.search(getHistogramIntervalRequest(params));

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts
View file

@ -19,7 +19,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
@ -32,7 +32,7 @@ const getHistogramRangeSteps = (min: number, max: number, steps: number) => {
};
export const getHistogramIntervalRequest = (
params: SearchServiceParams
params: SearchServiceFetchParams
): estypes.SearchRequest => ({
index: params.index,
body: {
@ -47,7 +47,7 @@ export const getHistogramIntervalRequest = (
export const fetchTransactionDurationHistogramRangeSteps = async (
esClient: ElasticsearchClient,
params: SearchServiceParams
params: SearchServiceFetchParams
): Promise<number[]> => {
const steps = 100;

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts
View file

@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
import { SIGNIFICANT_VALUE_DIGITS } from './constants';
@ -28,7 +28,7 @@ interface ResponseHit {
}
export const getTransactionDurationPercentilesRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
percents?: number[],
fieldName?: string,
fieldValue?: string
@ -58,7 +58,7 @@ export const getTransactionDurationPercentilesRequest = (
export const fetchTransactionDurationPercentiles = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
percents?: number[],
fieldName?: string,
fieldValue?: string

6
x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts
View file

@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
import type { ElasticsearchClient } from 'src/core/server';
import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
import { getQueryWithParams } from './get_query_with_params';
@ -27,7 +27,7 @@ interface ResponseHit {
}
export const getTransactionDurationRangesRequest = (
params: SearchServiceParams,
params: SearchServiceFetchParams,
rangesSteps: number[],
fieldName?: string,
fieldValue?: string
@ -65,7 +65,7 @@ export const getTransactionDurationRangesRequest = (
export const fetchTransactionDurationRanges = async (
esClient: ElasticsearchClient,
params: SearchServiceParams,
params: SearchServiceFetchParams,
rangesSteps: number[],
fieldName?: string,
fieldValue?: string

60
x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts
View file

@ -9,6 +9,8 @@ import type { estypes } from '@elastic/elasticsearch';
import { SearchStrategyDependencies } from 'src/plugins/data/server';
import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
import {
apmCorrelationsSearchStrategyProvider,
PartialSearchRequest,
@ -94,10 +96,19 @@ const clientSearchMock = (
};
};
const getApmIndicesMock = async () =>
({
// eslint-disable-next-line @typescript-eslint/naming-convention
'apm_oss.transactionIndices': 'apm-*',
} as ApmIndicesConfig);
describe('APM Correlations search strategy', () => {
describe('strategy interface', () => {
it('returns a custom search strategy with a `search` and `cancel` function', async () => {
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
getApmIndicesMock,
false
);
expect(typeof searchStrategy.search).toBe('function');
expect(typeof searchStrategy.cancel).toBe('function');
});
@ -106,12 +117,14 @@ describe('APM Correlations search strategy', () => {
describe('search', () => {
let mockClientFieldCaps: jest.Mock;
let mockClientSearch: jest.Mock;
let mockGetApmIndicesMock: jest.Mock;
let mockDeps: SearchStrategyDependencies;
let params: Required<PartialSearchRequest>['params'];
beforeEach(() => {
mockClientFieldCaps = jest.fn(clientFieldCapsMock);
mockClientSearch = jest.fn(clientSearchMock);
mockGetApmIndicesMock = jest.fn(getApmIndicesMock);
mockDeps = ({
esClient: {
asCurrentUser: {
@ -121,7 +134,6 @@ describe('APM Correlations search strategy', () => {
},
} as unknown) as SearchStrategyDependencies;
params = {
index: 'apm-*',
start: '2020',
end: '2021',
};
@ -130,7 +142,13 @@ describe('APM Correlations search strategy', () => {
describe('async functionality', () => {
describe('when no params are provided', () => {
it('throws an error', async () => {
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(0);
expect(() => searchStrategy.search({}, {}, mockDeps)).toThrow(
'Invalid request parameters.'
);
@ -139,8 +157,14 @@ describe('APM Correlations search strategy', () => {
describe('when no ID is provided', () => {
it('performs a client search with params', async () => {
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
await searchStrategy.search({ params }, {}, mockDeps).toPromise();
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
const [[request]] = mockClientSearch.mock.calls;
expect(request.index).toEqual('apm-*');
@ -179,7 +203,10 @@ describe('APM Correlations search strategy', () => {
describe('when an ID with params is provided', () => {
it('retrieves the current request', async () => {
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
const response = await searchStrategy
.search({ params }, {}, mockDeps)
.toPromise();
@ -190,6 +217,7 @@ describe('APM Correlations search strategy', () => {
.search({ id: searchStrategyId, params }, {}, mockDeps)
.toPromise();
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
expect(response2).toEqual(
expect.objectContaining({ id: searchStrategyId })
);
@ -201,11 +229,16 @@ describe('APM Correlations search strategy', () => {
mockClientSearch
.mockReset()
.mockRejectedValueOnce(new Error('client error'));
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
const response = await searchStrategy
.search({ params }, {}, mockDeps)
.toPromise();
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
expect(response).toEqual(
expect.objectContaining({ isRunning: true })
);
@ -213,11 +246,15 @@ describe('APM Correlations search strategy', () => {
});
it('triggers the subscription only once', async () => {
expect.assertions(1);
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
expect.assertions(2);
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
searchStrategy
.search({ params }, {}, mockDeps)
.subscribe((response) => {
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
expect(response).toEqual(
expect.objectContaining({ loaded: 0, isRunning: true })
);
@ -227,12 +264,16 @@ describe('APM Correlations search strategy', () => {
describe('response', () => {
it('sends an updated response on consecutive search calls', async () => {
const searchStrategy = await apmCorrelationsSearchStrategyProvider();
const searchStrategy = await apmCorrelationsSearchStrategyProvider(
mockGetApmIndicesMock,
false
);
const response1 = await searchStrategy
.search({ params }, {}, mockDeps)
.toPromise();
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
expect(typeof response1.id).toEqual('string');
expect(response1).toEqual(
expect.objectContaining({ loaded: 0, isRunning: true })
@ -244,6 +285,7 @@ describe('APM Correlations search strategy', () => {
.search({ id: response1.id, params }, {}, mockDeps)
.toPromise();
expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
expect(response2.id).toEqual(response1.id);
expect(response2).toEqual(
expect.objectContaining({ loaded: 100, isRunning: false })

16
x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts
View file

@ -19,6 +19,8 @@ import type {
SearchServiceValue,
} from '../../../../common/search_strategies/correlations/types';
import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
import { asyncSearchServiceProvider } from './async_search_service';
export type PartialSearchRequest = IKibanaSearchRequest<SearchServiceParams>;
@ -26,10 +28,10 @@ export type PartialSearchResponse = IKibanaSearchResponse<{
values: SearchServiceValue[];
}>;
export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
PartialSearchRequest,
PartialSearchResponse
> => {
export const apmCorrelationsSearchStrategyProvider = (
getApmIndices: () => Promise<ApmIndicesConfig>,
includeFrozen: boolean
): ISearchStrategy<PartialSearchRequest, PartialSearchResponse> => {
const asyncSearchServiceMap = new Map<
string,
ReturnType<typeof asyncSearchServiceProvider>
@ -65,7 +67,9 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
} else {
getAsyncSearchServiceState = asyncSearchServiceProvider(
deps.esClient.asCurrentUser,
request.params
getApmIndices,
request.params,
includeFrozen
);
}
@ -73,6 +77,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
const id = request.id ?? uuid();
const {
ccsWarning,
error,
log,
isRunning,
@ -102,6 +107,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
isRunning,
isPartial: isRunning,
rawResponse: {
ccsWarning,
log,
took,
values,

25
x-pack/plugins/apm/server/plugin.ts
View file

@ -16,6 +16,7 @@ import {
PluginInitializerContext,
} from 'src/core/server';
import { isEmpty, mapValues, once } from 'lodash';
import { SavedObjectsClient } from '../../../../src/core/server';
import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets';
import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map';
import { APMConfig, APMXPackConfig, APM_SERVER_FEATURE_ID } from '.';
@ -248,12 +249,24 @@ export class APMPlugin
});
// search strategies for async partial search results
if (plugins.data?.search?.registerSearchStrategy !== undefined) {
plugins.data.search.registerSearchStrategy(
'apmCorrelationsSearchStrategy',
apmCorrelationsSearchStrategyProvider()
);
}
core.getStartServices().then(([coreStart]) => {
(async () => {
const savedObjectsClient = new SavedObjectsClient(
coreStart.savedObjects.createInternalRepository()
);
plugins.data.search.registerSearchStrategy(
'apmCorrelationsSearchStrategy',
apmCorrelationsSearchStrategyProvider(
boundGetApmIndices,
await coreStart.uiSettings
.asScopedToClient(savedObjectsClient)
.get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN)
)
);
})();
});
return {
config$: mergedConfig$,
getApmIndices: boundGetApmIndices,

23
x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts
View file

@ -26,7 +26,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const getRequestBody = () => {
const partialSearchRequest: PartialSearchRequest = {
params: {
index: 'apm-*',
environment: 'ENVIRONMENT_ALL',
start: '2020',
end: '2021',
@ -141,7 +140,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
registry.when(
'Correlations latency_ml with data and opbeans-node args',
{ config: 'trial', archives: ['ml_8.0.0'] },
{ config: 'trial', archives: ['8.0.0'] },
() => {
// putting this into a single `it` because the responses depend on each other
it('queries the search strategy and returns results', async () => {
@ -235,30 +234,30 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const { rawResponse: finalRawResponse } = followUpResult;
expect(typeof finalRawResponse?.took).to.be('number');
expect(finalRawResponse?.percentileThresholdValue).to.be(1404927.875);
expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875);
expect(finalRawResponse?.overallHistogram.length).to.be(101);
expect(finalRawResponse?.values.length).to.eql(
1,
`Expected 1 identified correlations, got ${finalRawResponse?.values.length}.`
13,
`Expected 13 identified correlations, got ${finalRawResponse?.values.length}.`
);
expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([
'Fetched 95th percentile value of 1404927.875 based on 989 documents.',
'Fetched 95th percentile value of 1309695.875 based on 1244 documents.',
'Loaded histogram range steps.',
'Loaded overall histogram chart data.',
'Loaded percentiles.',
'Identified 67 fieldCandidates.',
'Identified 339 fieldValuePairs.',
'Loaded fractions and totalDocCount of 989.',
'Identified 1 significant correlations out of 339 field/value pairs.',
'Identified 69 fieldCandidates.',
'Identified 379 fieldValuePairs.',
'Loaded fractions and totalDocCount of 1244.',
'Identified 13 significant correlations out of 379 field/value pairs.',
]);
const correlation = finalRawResponse?.values[0];
expect(typeof correlation).to.be('object');
expect(correlation?.field).to.be('transaction.result');
expect(correlation?.value).to.be('success');
expect(correlation?.correlation).to.be(0.37418510688551887);
expect(correlation?.ksTest).to.be(1.1238496968312214e-10);
expect(correlation?.correlation).to.be(0.6275246559191225);
expect(correlation?.ksTest).to.be(4.806503252860024e-13);
expect(correlation?.histogram.length).to.be(101);
});
}

15
x-pack/test/functional/apps/apm/correlations/index.ts Normal file
View file

@ -0,0 +1,15 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ loadTestFile }: FtrProviderContext) {
describe('correlations', function () {
this.tags('skipFirefox');
loadTestFile(require.resolve('./latency_correlations'));
});
}

139
x-pack/test/functional/apps/apm/correlations/latency_correlations.ts Normal file
View file

@ -0,0 +1,139 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../../ftr_provider_context';
export default function ({ getPageObjects, getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const find = getService('find');
const retry = getService('retry');
const spacesService = getService('spaces');
const PageObjects = getPageObjects(['common', 'error', 'timePicker', 'security']);
const testSubjects = getService('testSubjects');
const appsMenu = getService('appsMenu');
const testData = { serviceName: 'opbeans-go' };
describe('latency correlations', () => {
describe('space with no features disabled', () => {
before(async () => {
await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
await spacesService.create({
id: 'custom_space',
name: 'custom_space',
disabledFeatures: [],
});
});
after(async () => {
await spacesService.delete('custom_space');
});
it('shows apm navlink', async () => {
await PageObjects.common.navigateToApp('home', {
basePath: '/s/custom_space',
});
const navLinks = (await appsMenu.readLinks()).map((link) => link.text);
expect(navLinks).to.contain('APM');
});
it('can navigate to APM app', async () => {
await PageObjects.common.navigateToApp('apm');
await retry.try(async () => {
await testSubjects.existOrFail('apmMainContainer', {
timeout: 10000,
});
const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer');
const apmMainContainerTextItems = apmMainContainerText[0].split('\n');
expect(apmMainContainerTextItems).to.contain('No services found');
});
});
it('sets the timePicker to return data', async () => {
await PageObjects.timePicker.timePickerExists();
const fromTime = 'Jul 29, 2019 @ 00:00:00.000';
const toTime = 'Jul 30, 2019 @ 00:00:00.000';
await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
await retry.try(async () => {
const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer');
const apmMainContainerTextItems = apmMainContainerText[0].split('\n');
expect(apmMainContainerTextItems).to.not.contain('No services found');
expect(apmMainContainerTextItems).to.contain('opbeans-go');
expect(apmMainContainerTextItems).to.contain('opbeans-node');
expect(apmMainContainerTextItems).to.contain('opbeans-ruby');
expect(apmMainContainerTextItems).to.contain('opbeans-python');
expect(apmMainContainerTextItems).to.contain('opbeans-dotnet');
expect(apmMainContainerTextItems).to.contain('opbeans-java');
expect(apmMainContainerTextItems).to.contain('development');
const items = await testSubjects.findAll('apmServiceListAppLink');
expect(items.length).to.be(6);
});
});
it(`navigates to the 'opbeans-go' service overview page`, async function () {
await find.clickByDisplayedLinkText(testData.serviceName);
await retry.try(async () => {
const apmMainTemplateHeaderServiceName = await testSubjects.getVisibleTextAll(
'apmMainTemplateHeaderServiceName'
);
expect(apmMainTemplateHeaderServiceName).to.contain('opbeans-go');
});
});
it('shows the correlations flyout', async function () {
await testSubjects.click('apmViewCorrelationsButton');
await retry.try(async () => {
await testSubjects.existOrFail('apmCorrelationsFlyout', {
timeout: 10000,
});
const apmCorrelationsFlyoutHeader = await testSubjects.getVisibleText(
'apmCorrelationsFlyoutHeader'
);
expect(apmCorrelationsFlyoutHeader).to.contain('Correlations BETA');
});
});
it('loads the correlation results', async function () {
await retry.try(async () => {
// Assert that the data fully loaded to 100%
const apmCorrelationsLatencyCorrelationsProgressTitle = await testSubjects.getVisibleText(
'apmCorrelationsLatencyCorrelationsProgressTitle'
);
expect(apmCorrelationsLatencyCorrelationsProgressTitle).to.be('Progress: 100%');
// Assert that the Correlations Chart and its header are present
const apmCorrelationsLatencyCorrelationsChartTitle = await testSubjects.getVisibleText(
'apmCorrelationsLatencyCorrelationsChartTitle'
);
expect(apmCorrelationsLatencyCorrelationsChartTitle).to.be(
`Latency distribution for ${testData.serviceName}`
);
await testSubjects.existOrFail('apmCorrelationsChart', {
timeout: 10000,
});
// Assert that results for the given service didn't find any correlations
const apmCorrelationsTable = await testSubjects.getVisibleText('apmCorrelationsTable');
expect(apmCorrelationsTable).to.be('No significant correlations found');
});
});
});
});
}

1
x-pack/test/functional/apps/apm/index.ts
View file

@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
describe('APM specs', function () {
this.tags('ciGroup6');
loadTestFile(require.resolve('./feature_controls'));
loadTestFile(require.resolve('./correlations'));
});
}