[ML] Fix custom index name settings, functional tests for APM Latency Correlation. (#105200)

Adds first functional tests for APM Latency Correlations code.
- Fix: The log log chart's Y axis would only start at 1 by default which hides results for small datasets like the ones used in these tests. Starting the axis at 0 isn't supported to log based ones so we're setting it to just a small value above 0.
- Fix: Instead of the hard coded apm-* index we passed on from the client, we now correctly consider APM's custom settings.

x-pack/plugins/apm/common/search_strategies/correlations/types.ts

@@ -19,7 +19,6 @@ export interface ResponseHit {
 }
 
 export interface SearchServiceParams {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -31,6 +30,10 @@ export interface SearchServiceParams {
   percentileThresholdValue?: number;
 }
 
+export interface SearchServiceFetchParams extends SearchServiceParams {
+  index: string;
+}
+
 export interface SearchServiceValue {
   histogram: HistogramItem[];
   value: string;

x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx

@@ -70,6 +70,11 @@ const chartTheme: PartialTheme = {
   },
 };
 
+// Log based axis cannot start a 0. Use a small positive number instead.
+const yAxisDomain = {
+  min: 0.00001,
+};
+
 interface CorrelationsChartProps {
   field?: string;
   value?: string;
@@ -140,7 +145,10 @@ export function CorrelationsChart({
   const histogram = replaceHistogramDotsWithBars(originalHistogram);
 
   return (
-    <div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
+    <div
+      data-test-subj="apmCorrelationsChart"
+      style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}
+    >
       <Chart
         size={{
           height: '250px',
@@ -168,6 +176,7 @@ export function CorrelationsChart({
         />
         <Axis
           id="y-axis"
+          domain={yAxisDomain}
           title={i18n.translate(
             'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
             { defaultMessage: '# transactions' }

x-pack/plugins/apm/public/components/app/correlations/index.tsx

@@ -134,6 +134,7 @@ export function Correlations() {
   return (
     <>
       <EuiButton
+        data-test-subj="apmViewCorrelationsButton"
         fill
         onClick={() => {
           setIsFlyoutVisible(true);
@@ -147,13 +148,17 @@ export function Correlations() {
       {isFlyoutVisible && (
         <EuiPortal>
           <EuiFlyout
+            data-test-subj="apmCorrelationsFlyout"
             size="l"
             ownFocus
             onClose={() => setIsFlyoutVisible(false)}
           >
             <EuiFlyoutHeader hasBorder aria-labelledby="correlations-flyout">
               <EuiTitle>
-                <h2 id="correlations-flyout">
+                <h2
+                  data-test-subj="apmCorrelationsFlyoutHeader"
+                  id="correlations-flyout"
+                >
                   {CORRELATIONS_TITLE}
                   &nbsp;
                   <EuiBetaBadge

x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx

@@ -8,6 +8,10 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import {
+  EuiCallOut,
+  EuiCode,
+  EuiAccordion,
+  EuiPanel,
   EuiIcon,
   EuiBasicTableColumn,
   EuiButton,
@@ -34,7 +38,10 @@ import {
 } from './correlations_table';
 import { useCorrelations } from './use_correlations';
 import { push } from '../../shared/Links/url_helpers';
-import { useUiTracker } from '../../../../../observability/public';
+import {
+  enableInspectEsQueries,
+  useUiTracker,
+} from '../../../../../observability/public';
 import { asPreciseDecimal } from '../../../../common/utils/formatters';
 import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context';
 import { LatencyCorrelationsHelpPopover } from './ml_latency_correlations_help_popover';
@@ -58,7 +65,7 @@ interface MlCorrelationsTerms {
 
 export function MlLatencyCorrelations({ onClose }: Props) {
   const {
-    core: { notifications },
+    core: { notifications, uiSettings },
   } = useApmPluginContext();
 
   const { serviceName, transactionType } = useApmServiceContext();
@@ -66,7 +73,11 @@ export function MlLatencyCorrelations({ onClose }: Props) {
 
   const { environment, kuery, transactionName, start, end } = urlParams;
 
+  const displayLog = uiSettings.get<boolean>(enableInspectEsQueries);
+
   const {
+    ccsWarning,
+    log,
     error,
     histograms,
     percentileThresholdValue,
@@ -76,7 +87,6 @@ export function MlLatencyCorrelations({ onClose }: Props) {
     cancelFetch,
     overallHistogram: originalOverallHistogram,
   } = useCorrelations({
-    index: 'apm-*',
     ...{
       ...{
         environment,
@@ -286,9 +296,10 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiFlexGroup direction="column" gutterSize="none">
-            <EuiFlexItem>
+            <EuiFlexItem data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle">
               <EuiText size="xs" color="subdued">
                 <FormattedMessage
+                  data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle"
                   id="xpack.apm.correlations.latencyCorrelations.progressTitle"
                   defaultMessage="Progress: {progress}%"
                   values={{ progress: Math.round(progress * 100) }}
@@ -313,11 +324,35 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </EuiFlexItem>
       </EuiFlexGroup>
 
+      {ccsWarning && (
+        <>
+          <EuiSpacer size="m" />
+          <EuiCallOut
+            title={i18n.translate(
+              'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutTitle',
+              {
+                defaultMessage: 'Cross-cluster search compatibility',
+              }
+            )}
+            color="warning"
+          >
+            <p>
+              {i18n.translate(
+                'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutBody',
+                {
+                  defaultMessage:
+                    'Data for the correlation analysis could not be fully retrieved. This feature is supported only for 7.14 and later versions.',
+                }
+              )}
+            </p>
+          </EuiCallOut>
+        </>
+      )}
       <EuiSpacer size="m" />
       {overallHistogram !== undefined ? (
         <>
           <EuiTitle size="xxs">
-            <h4>
+            <h4 data-test-subj="apmCorrelationsLatencyCorrelationsChartTitle">
               {i18n.translate(
                 'xpack.apm.correlations.latencyCorrelations.chartTitle',
                 {
@@ -341,32 +376,58 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </>
       ) : null}
 
-      {histograms.length > 0 && selectedHistogram !== undefined && (
+      <div data-test-subj="apmCorrelationsTable">
-        <CorrelationsTable
+        {histograms.length > 0 && selectedHistogram !== undefined && (
-          // @ts-ignore correlations don't have the same column format other tables have
+          <CorrelationsTable
-          columns={mlCorrelationColumns}
+            // @ts-ignore correlations don't have the same column format other tables have
-          // @ts-expect-error correlations don't have the same significant term other tables have
+            columns={mlCorrelationColumns}
-          significantTerms={histogramTerms}
+            // @ts-expect-error correlations don't have the same significant term other tables have
-          status={FETCH_STATUS.SUCCESS}
+            significantTerms={histogramTerms}
-          setSelectedSignificantTerm={setSelectedSignificantTerm}
+            status={FETCH_STATUS.SUCCESS}
-          selectedTerm={{
+            setSelectedSignificantTerm={setSelectedSignificantTerm}
-            fieldName: selectedHistogram.field,
+            selectedTerm={{
-            fieldValue: selectedHistogram.value,
+              fieldName: selectedHistogram.field,
-          }}
+              fieldValue: selectedHistogram.value,
-          onFilter={onClose}
+            }}
-        />
+            onFilter={onClose}
+          />
+        )}
+        {histograms.length < 1 && progress > 0.99 ? (
+          <>
+            <EuiSpacer size="m" />
+            <EuiText textAlign="center">
+              <FormattedMessage
+                id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
+                defaultMessage="No significant correlations found"
+              />
+            </EuiText>
+          </>
+        ) : null}
+      </div>
+      {log.length > 0 && displayLog && (
+        <EuiAccordion
+          id="accordion1"
+          buttonContent={i18n.translate(
+            'xpack.apm.correlations.latencyCorrelations.logButtonContent',
+            {
+              defaultMessage: 'Log',
+            }
+          )}
+        >
+          <EuiPanel color="subdued">
+            {log.map((d, i) => {
+              const splitItem = d.split(': ');
+              return (
+                <p key={i}>
+                  <small>
+                    <EuiCode>{splitItem[0]}</EuiCode> {splitItem[1]}
+                  </small>
+                </p>
+              );
+            })}
+          </EuiPanel>
+        </EuiAccordion>
       )}
-      {histograms.length < 1 && progress > 0.99 ? (
-        <>
-          <EuiSpacer size="m" />
-          <EuiText textAlign="center">
-            <FormattedMessage
-              id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
-              defaultMessage="No significant correlations found"
-            />
-          </EuiText>
-        </>
-      ) : null}
     </>
   );
 }

x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts

@@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'
 import { ApmPluginStartDeps } from '../../../plugin';
 
 interface CorrelationsOptions {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -37,6 +36,7 @@ interface RawResponse {
   values: SearchServiceValue[];
   overallHistogram: HistogramItem[];
   log: string[];
+  ccsWarning: boolean;
 }
 
 export const useCorrelations = (params: CorrelationsOptions) => {
@@ -106,6 +106,8 @@ export const useCorrelations = (params: CorrelationsOptions) => {
   };
 
   return {
+    ccsWarning: rawResponse?.ccsWarning ?? false,
+    log: rawResponse?.log ?? [],
     error,
     histograms: rawResponse?.values ?? [],
     percentileThresholdValue:

x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx

@@ -117,6 +117,7 @@ export function getServiceColumns({
               )}
               <EuiFlexItem className="apmServiceList__serviceNameContainer">
                 <AppLink
+                  data-test-subj="apmServiceListAppLink"
                   serviceName={serviceName}
                   transactionType={transactionType}
                   className="eui-textTruncate"

x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx

@@ -93,7 +93,9 @@ function TemplateWithContext({
               <EuiFlexGroup alignItems="center">
                 <EuiFlexItem grow={false}>
                   <EuiTitle size="l">
-                    <>{serviceName}</>
+                    <h1 data-test-subj="apmMainTemplateHeaderServiceName">
+                      {serviceName}
+                    </h1>
                   </EuiTitle>
                 </EuiFlexItem>
                 <EuiFlexItem grow={false}>

x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts

@@ -7,6 +7,7 @@
 
 import { shuffle, range } from 'lodash';
 import type { ElasticsearchClient } from 'src/core/server';
+import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
 import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
 import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
 import { fetchTransactionDurationPercentiles } from './query_percentiles';
@@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
 import type {
   AsyncSearchProviderProgress,
   SearchServiceParams,
+  SearchServiceFetchParams,
   SearchServiceValue,
 } from '../../../../common/search_strategies/correlations/types';
 import { computeExpectationsAndRanges } from './utils/aggregation_utils';
@@ -28,11 +30,14 @@ const currentTimeAsString = () => new Date().toISOString();
 
 export const asyncSearchServiceProvider = (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  getApmIndices: () => Promise<ApmIndicesConfig>,
+  searchServiceParams: SearchServiceParams,
+  includeFrozen: boolean
 ) => {
   let isCancelled = false;
   let isRunning = true;
   let error: Error;
+  let ccsWarning = false;
   const log: string[] = [];
   const logMessage = (message: string) =>
     log.push(`${currentTimeAsString()}: ${message}`);
@@ -63,7 +68,15 @@ export const asyncSearchServiceProvider = (
   };
 
   const fetchCorrelations = async () => {
+    let params: SearchServiceFetchParams | undefined;
+
     try {
+      const indices = await getApmIndices();
+      params = {
+        ...searchServiceParams,
+        index: indices['apm_oss.transactionIndices'],
+      };
+
       // 95th percentile to be displayed as a marker in the log log chart
       const {
         totalDocs,
@@ -172,7 +185,7 @@ export const asyncSearchServiceProvider = (
 
       async function* fetchTransactionDurationHistograms() {
         for (const item of shuffle(fieldValuePairs)) {
-          if (item === undefined || isCancelled) {
+          if (params === undefined || item === undefined || isCancelled) {
             isRunning = false;
             return;
           }
@@ -222,10 +235,15 @@ export const asyncSearchServiceProvider = (
               yield undefined;
             }
           } catch (e) {
-            // don't fail the whole process for individual correlation queries, just add the error to the internal log.
+            // don't fail the whole process for individual correlation queries,
+            // just add the error to the internal log and check if we'd want to set the
+            // cross-cluster search compatibility warning to true.
             logMessage(
               `Failed to fetch correlation/kstest for '${item.field}/${item.value}'`
             );
+            if (params?.index.includes(':')) {
+              ccsWarning = true;
+            }
             yield undefined;
           }
         }
@@ -247,6 +265,10 @@ export const asyncSearchServiceProvider = (
       error = e;
     }
 
+    if (error !== undefined && params?.index.includes(':')) {
+      ccsWarning = true;
+    }
+
     isRunning = false;
   };
 
@@ -256,6 +278,7 @@ export const asyncSearchServiceProvider = (
     const sortedValues = values.sort((a, b) => b.correlation - a.correlation);
 
     return {
+      ccsWarning,
       error,
       log,
       isRunning,

x-pack/plugins/apm/server/lib/search_strategies/correlations/get_query_with_params.ts

@@ -11,7 +11,7 @@ import { pipe } from 'fp-ts/lib/pipeable';
 import * as t from 'io-ts';
 import { failure } from 'io-ts/lib/PathReporter';
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 import { rangeRt } from '../../../routes/default_api_types';
 import { getCorrelationsFilters } from '../../correlations/get_filters';
 import { Setup, SetupTimeRange } from '../../helpers/setup_request';
@@ -40,7 +40,7 @@ export const getTermsQuery = (
 };
 
 interface QueryParams {
-  params: SearchServiceParams;
+  params: SearchServiceFetchParams;
   fieldName?: string;
   fieldValue?: string;
 }

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_correlation.ts

@@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
@@ -40,7 +40,7 @@ export interface BucketCorrelation {
 }
 
 export const getTransactionDurationCorrelationRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   expectations: number[],
   ranges: estypes.AggregationsAggregationRange[],
   fractions: number[],
@@ -95,7 +95,7 @@ export const getTransactionDurationCorrelationRequest = (
 
 export const fetchTransactionDurationCorrelation = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   expectations: number[],
   ranges: estypes.AggregationsAggregationRange[],
   fractions: number[],

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_candidates.ts

@@ -9,7 +9,7 @@ import type { estypes } from '@elastic/elasticsearch';
 
 import type { ElasticsearchClient } from 'src/core/server';
 
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 import { Field } from './query_field_value_pairs';
@@ -37,7 +37,7 @@ export const hasPrefixToInclude = (fieldName: string) => {
 };
 
 export const getRandomDocsRequest = (
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): estypes.SearchRequest => ({
   index: params.index,
   body: {
@@ -56,7 +56,7 @@ export const getRandomDocsRequest = (
 
 export const fetchTransactionDurationFieldCandidates = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): Promise<{ fieldCandidates: Field[] }> => {
   const { index } = params;
   // Get all fields with keyword mapping

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_field_value_pairs.ts

@@ -11,7 +11,7 @@ import type { estypes } from '@elastic/elasticsearch';
 
 import type {
   AsyncSearchProviderProgress,
-  SearchServiceParams,
+  SearchServiceFetchParams,
 } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
@@ -26,7 +26,7 @@ type FieldValuePairs = FieldValuePair[];
 export type Field = string;
 
 export const getTermsAggRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   fieldName: string
 ): estypes.SearchRequest => ({
   index: params.index,
@@ -46,7 +46,7 @@ export const getTermsAggRequest = (
 
 export const fetchTransactionDurationFieldValuePairs = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   fieldCandidates: Field[],
   progress: AsyncSearchProviderProgress
 ): Promise<FieldValuePairs> => {

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_fractions.ts

@@ -7,12 +7,12 @@
 
 import { ElasticsearchClient } from 'kibana/server';
 import { estypes } from '@elastic/elasticsearch';
-import { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 import { getQueryWithParams } from './get_query_with_params';
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
 
 export const getTransactionDurationRangesRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   ranges: estypes.AggregationsAggregationRange[]
 ): estypes.SearchRequest => ({
   index: params.index,
@@ -35,7 +35,7 @@ export const getTransactionDurationRangesRequest = (
  */
 export const fetchTransactionDurationFractions = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   ranges: estypes.AggregationsAggregationRange[]
 ): Promise<{ fractions: number[]; totalDocCount: number }> => {
   const resp = await esClient.search(

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram.ts

@@ -13,13 +13,13 @@ import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldname
 import type {
   HistogramItem,
   ResponseHit,
-  SearchServiceParams,
+  SearchServiceFetchParams,
 } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
 export const getTransactionDurationHistogramRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   interval: number,
   fieldName?: string,
   fieldValue?: string
@@ -42,7 +42,7 @@ export const getTransactionDurationHistogramRequest = (
 
 export const fetchTransactionDurationHistogram = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   interval: number,
   fieldName?: string,
   fieldValue?: string

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_interval.ts

@@ -10,14 +10,14 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
 const HISTOGRAM_INTERVALS = 1000;
 
 export const getHistogramIntervalRequest = (
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): estypes.SearchRequest => ({
   index: params.index,
   body: {
@@ -32,7 +32,7 @@ export const getHistogramIntervalRequest = (
 
 export const fetchTransactionDurationHistogramInterval = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): Promise<number> => {
   const resp = await esClient.search(getHistogramIntervalRequest(params));
 

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_histogram_range_steps.ts

@@ -19,7 +19,7 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
@@ -32,7 +32,7 @@ const getHistogramRangeSteps = (min: number, max: number, steps: number) => {
 };
 
 export const getHistogramIntervalRequest = (
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): estypes.SearchRequest => ({
   index: params.index,
   body: {
@@ -47,7 +47,7 @@ export const getHistogramIntervalRequest = (
 
 export const fetchTransactionDurationHistogramRangeSteps = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  params: SearchServiceFetchParams
 ): Promise<number[]> => {
   const steps = 100;
 

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_percentiles.ts

@@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 import { SIGNIFICANT_VALUE_DIGITS } from './constants';
@@ -28,7 +28,7 @@ interface ResponseHit {
 }
 
 export const getTransactionDurationPercentilesRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   percents?: number[],
   fieldName?: string,
   fieldValue?: string
@@ -58,7 +58,7 @@ export const getTransactionDurationPercentilesRequest = (
 
 export const fetchTransactionDurationPercentiles = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   percents?: number[],
   fieldName?: string,
   fieldValue?: string

x-pack/plugins/apm/server/lib/search_strategies/correlations/query_ranges.ts

@@ -10,7 +10,7 @@ import type { estypes } from '@elastic/elasticsearch';
 import type { ElasticsearchClient } from 'src/core/server';
 
 import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames';
-import type { SearchServiceParams } from '../../../../common/search_strategies/correlations/types';
+import type { SearchServiceFetchParams } from '../../../../common/search_strategies/correlations/types';
 
 import { getQueryWithParams } from './get_query_with_params';
 
@@ -27,7 +27,7 @@ interface ResponseHit {
 }
 
 export const getTransactionDurationRangesRequest = (
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   rangesSteps: number[],
   fieldName?: string,
   fieldValue?: string
@@ -65,7 +65,7 @@ export const getTransactionDurationRangesRequest = (
 
 export const fetchTransactionDurationRanges = async (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams,
+  params: SearchServiceFetchParams,
   rangesSteps: number[],
   fieldName?: string,
   fieldValue?: string

x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.test.ts

@@ -9,6 +9,8 @@ import type { estypes } from '@elastic/elasticsearch';
 
 import { SearchStrategyDependencies } from 'src/plugins/data/server';
 
+import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+
 import {
   apmCorrelationsSearchStrategyProvider,
   PartialSearchRequest,
@@ -94,10 +96,19 @@ const clientSearchMock = (
   };
 };
 
+const getApmIndicesMock = async () =>
+  ({
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    'apm_oss.transactionIndices': 'apm-*',
+  } as ApmIndicesConfig);
+
 describe('APM Correlations search strategy', () => {
   describe('strategy interface', () => {
     it('returns a custom search strategy with a `search` and `cancel` function', async () => {
-      const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+      const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+        getApmIndicesMock,
+        false
+      );
       expect(typeof searchStrategy.search).toBe('function');
       expect(typeof searchStrategy.cancel).toBe('function');
     });
@@ -106,12 +117,14 @@ describe('APM Correlations search strategy', () => {
   describe('search', () => {
     let mockClientFieldCaps: jest.Mock;
     let mockClientSearch: jest.Mock;
+    let mockGetApmIndicesMock: jest.Mock;
     let mockDeps: SearchStrategyDependencies;
     let params: Required<PartialSearchRequest>['params'];
 
     beforeEach(() => {
       mockClientFieldCaps = jest.fn(clientFieldCapsMock);
       mockClientSearch = jest.fn(clientSearchMock);
+      mockGetApmIndicesMock = jest.fn(getApmIndicesMock);
       mockDeps = ({
         esClient: {
           asCurrentUser: {
@@ -121,7 +134,6 @@ describe('APM Correlations search strategy', () => {
         },
       } as unknown) as SearchStrategyDependencies;
       params = {
-        index: 'apm-*',
         start: '2020',
         end: '2021',
       };
@@ -130,7 +142,13 @@ describe('APM Correlations search strategy', () => {
     describe('async functionality', () => {
       describe('when no params are provided', () => {
         it('throws an error', async () => {
-          const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+          const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+            mockGetApmIndicesMock,
+            false
+          );
+
+          expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(0);
+
           expect(() => searchStrategy.search({}, {}, mockDeps)).toThrow(
             'Invalid request parameters.'
           );
@@ -139,8 +157,14 @@ describe('APM Correlations search strategy', () => {
 
       describe('when no ID is provided', () => {
         it('performs a client search with params', async () => {
-          const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+          const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+            mockGetApmIndicesMock,
+            false
+          );
           await searchStrategy.search({ params }, {}, mockDeps).toPromise();
+
+          expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
+
           const [[request]] = mockClientSearch.mock.calls;
 
           expect(request.index).toEqual('apm-*');
@@ -179,7 +203,10 @@ describe('APM Correlations search strategy', () => {
 
       describe('when an ID with params is provided', () => {
         it('retrieves the current request', async () => {
-          const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+          const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+            mockGetApmIndicesMock,
+            false
+          );
           const response = await searchStrategy
             .search({ params }, {}, mockDeps)
             .toPromise();
@@ -190,6 +217,7 @@ describe('APM Correlations search strategy', () => {
             .search({ id: searchStrategyId, params }, {}, mockDeps)
             .toPromise();
 
+          expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
           expect(response2).toEqual(
             expect.objectContaining({ id: searchStrategyId })
           );
@@ -201,11 +229,16 @@ describe('APM Correlations search strategy', () => {
           mockClientSearch
             .mockReset()
             .mockRejectedValueOnce(new Error('client error'));
-          const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+          const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+            mockGetApmIndicesMock,
+            false
+          );
           const response = await searchStrategy
             .search({ params }, {}, mockDeps)
             .toPromise();
 
+          expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
+
           expect(response).toEqual(
             expect.objectContaining({ isRunning: true })
           );
@@ -213,11 +246,15 @@ describe('APM Correlations search strategy', () => {
       });
 
       it('triggers the subscription only once', async () => {
-        expect.assertions(1);
+        expect.assertions(2);
-        const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+        const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+          mockGetApmIndicesMock,
+          false
+        );
         searchStrategy
           .search({ params }, {}, mockDeps)
           .subscribe((response) => {
+            expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
             expect(response).toEqual(
               expect.objectContaining({ loaded: 0, isRunning: true })
             );
@@ -227,12 +264,16 @@ describe('APM Correlations search strategy', () => {
 
     describe('response', () => {
       it('sends an updated response on consecutive search calls', async () => {
-        const searchStrategy = await apmCorrelationsSearchStrategyProvider();
+        const searchStrategy = await apmCorrelationsSearchStrategyProvider(
+          mockGetApmIndicesMock,
+          false
+        );
 
         const response1 = await searchStrategy
           .search({ params }, {}, mockDeps)
           .toPromise();
 
+        expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
         expect(typeof response1.id).toEqual('string');
         expect(response1).toEqual(
           expect.objectContaining({ loaded: 0, isRunning: true })
@@ -244,6 +285,7 @@ describe('APM Correlations search strategy', () => {
           .search({ id: response1.id, params }, {}, mockDeps)
           .toPromise();
 
+        expect(mockGetApmIndicesMock).toHaveBeenCalledTimes(1);
         expect(response2.id).toEqual(response1.id);
         expect(response2).toEqual(
           expect.objectContaining({ loaded: 100, isRunning: false })

x-pack/plugins/apm/server/lib/search_strategies/correlations/search_strategy.ts

@@ -19,6 +19,8 @@ import type {
   SearchServiceValue,
 } from '../../../../common/search_strategies/correlations/types';
 
+import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
+
 import { asyncSearchServiceProvider } from './async_search_service';
 
 export type PartialSearchRequest = IKibanaSearchRequest<SearchServiceParams>;
@@ -26,10 +28,10 @@ export type PartialSearchResponse = IKibanaSearchResponse<{
   values: SearchServiceValue[];
 }>;
 
-export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
+export const apmCorrelationsSearchStrategyProvider = (
-  PartialSearchRequest,
+  getApmIndices: () => Promise<ApmIndicesConfig>,
-  PartialSearchResponse
+  includeFrozen: boolean
-> => {
+): ISearchStrategy<PartialSearchRequest, PartialSearchResponse> => {
   const asyncSearchServiceMap = new Map<
     string,
     ReturnType<typeof asyncSearchServiceProvider>
@@ -65,7 +67,9 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
       } else {
         getAsyncSearchServiceState = asyncSearchServiceProvider(
           deps.esClient.asCurrentUser,
-          request.params
+          getApmIndices,
+          request.params,
+          includeFrozen
         );
       }
 
@@ -73,6 +77,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
       const id = request.id ?? uuid();
 
       const {
+        ccsWarning,
         error,
         log,
         isRunning,
@@ -102,6 +107,7 @@ export const apmCorrelationsSearchStrategyProvider = (): ISearchStrategy<
         isRunning,
         isPartial: isRunning,
         rawResponse: {
+          ccsWarning,
           log,
           took,
           values,

x-pack/plugins/apm/server/plugin.ts

@@ -16,6 +16,7 @@ import {
   PluginInitializerContext,
 } from 'src/core/server';
 import { isEmpty, mapValues, once } from 'lodash';
+import { SavedObjectsClient } from '../../../../src/core/server';
 import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../rule_registry/common/assets';
 import { mappingFromFieldMap } from '../../rule_registry/common/mapping_from_field_map';
 import { APMConfig, APMXPackConfig, APM_SERVER_FEATURE_ID } from '.';
@@ -248,12 +249,24 @@ export class APMPlugin
     });
 
     // search strategies for async partial search results
-    if (plugins.data?.search?.registerSearchStrategy !== undefined) {
+    core.getStartServices().then(([coreStart]) => {
-      plugins.data.search.registerSearchStrategy(
+      (async () => {
-        'apmCorrelationsSearchStrategy',
+        const savedObjectsClient = new SavedObjectsClient(
-        apmCorrelationsSearchStrategyProvider()
+          coreStart.savedObjects.createInternalRepository()
-      );
+        );
-    }
+
+        plugins.data.search.registerSearchStrategy(
+          'apmCorrelationsSearchStrategy',
+          apmCorrelationsSearchStrategyProvider(
+            boundGetApmIndices,
+            await coreStart.uiSettings
+              .asScopedToClient(savedObjectsClient)
+              .get(UI_SETTINGS.SEARCH_INCLUDE_FROZEN)
+          )
+        );
+      })();
+    });
+
     return {
       config$: mergedConfig$,
       getApmIndices: boundGetApmIndices,

x-pack/test/apm_api_integration/tests/correlations/latency_ml.ts

@@ -26,7 +26,6 @@ export default function ApiTest({ getService }: FtrProviderContext) {
   const getRequestBody = () => {
     const partialSearchRequest: PartialSearchRequest = {
       params: {
-        index: 'apm-*',
         environment: 'ENVIRONMENT_ALL',
         start: '2020',
         end: '2021',
@@ -141,7 +140,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
 
   registry.when(
     'Correlations latency_ml with data and opbeans-node args',
-    { config: 'trial', archives: ['ml_8.0.0'] },
+    { config: 'trial', archives: ['8.0.0'] },
     () => {
       // putting this into a single `it` because the responses depend on each other
       it('queries the search strategy and returns results', async () => {
@@ -235,30 +234,30 @@ export default function ApiTest({ getService }: FtrProviderContext) {
         const { rawResponse: finalRawResponse } = followUpResult;
 
         expect(typeof finalRawResponse?.took).to.be('number');
-        expect(finalRawResponse?.percentileThresholdValue).to.be(1404927.875);
+        expect(finalRawResponse?.percentileThresholdValue).to.be(1309695.875);
         expect(finalRawResponse?.overallHistogram.length).to.be(101);
 
         expect(finalRawResponse?.values.length).to.eql(
-          1,
+          13,
-          `Expected 1 identified correlations, got ${finalRawResponse?.values.length}.`
+          `Expected 13 identified correlations, got ${finalRawResponse?.values.length}.`
         );
         expect(finalRawResponse?.log.map((d: string) => d.split(': ')[1])).to.eql([
-          'Fetched 95th percentile value of 1404927.875 based on 989 documents.',
+          'Fetched 95th percentile value of 1309695.875 based on 1244 documents.',
           'Loaded histogram range steps.',
           'Loaded overall histogram chart data.',
           'Loaded percentiles.',
-          'Identified 67 fieldCandidates.',
+          'Identified 69 fieldCandidates.',
-          'Identified 339 fieldValuePairs.',
+          'Identified 379 fieldValuePairs.',
-          'Loaded fractions and totalDocCount of 989.',
+          'Loaded fractions and totalDocCount of 1244.',
-          'Identified 1 significant correlations out of 339 field/value pairs.',
+          'Identified 13 significant correlations out of 379 field/value pairs.',
         ]);
 
         const correlation = finalRawResponse?.values[0];
         expect(typeof correlation).to.be('object');
         expect(correlation?.field).to.be('transaction.result');
         expect(correlation?.value).to.be('success');
-        expect(correlation?.correlation).to.be(0.37418510688551887);
+        expect(correlation?.correlation).to.be(0.6275246559191225);
-        expect(correlation?.ksTest).to.be(1.1238496968312214e-10);
+        expect(correlation?.ksTest).to.be(4.806503252860024e-13);
         expect(correlation?.histogram.length).to.be(101);
       });
     }

x-pack/test/functional/apps/apm/correlations/index.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ loadTestFile }: FtrProviderContext) {
+  describe('correlations', function () {
+    this.tags('skipFirefox');
+    loadTestFile(require.resolve('./latency_correlations'));
+  });
+}

x-pack/test/functional/apps/apm/correlations/latency_correlations.ts

@@ -0,0 +1,139 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import expect from '@kbn/expect';
+import { FtrProviderContext } from '../../../ftr_provider_context';
+
+export default function ({ getPageObjects, getService }: FtrProviderContext) {
+  const esArchiver = getService('esArchiver');
+  const find = getService('find');
+  const retry = getService('retry');
+  const spacesService = getService('spaces');
+  const PageObjects = getPageObjects(['common', 'error', 'timePicker', 'security']);
+  const testSubjects = getService('testSubjects');
+  const appsMenu = getService('appsMenu');
+
+  const testData = { serviceName: 'opbeans-go' };
+
+  describe('latency correlations', () => {
+    describe('space with no features disabled', () => {
+      before(async () => {
+        await esArchiver.load('x-pack/test/functional/es_archives/infra/8.0.0/metrics_and_apm');
+        await spacesService.create({
+          id: 'custom_space',
+          name: 'custom_space',
+          disabledFeatures: [],
+        });
+      });
+
+      after(async () => {
+        await spacesService.delete('custom_space');
+      });
+
+      it('shows apm navlink', async () => {
+        await PageObjects.common.navigateToApp('home', {
+          basePath: '/s/custom_space',
+        });
+        const navLinks = (await appsMenu.readLinks()).map((link) => link.text);
+        expect(navLinks).to.contain('APM');
+      });
+
+      it('can navigate to APM app', async () => {
+        await PageObjects.common.navigateToApp('apm');
+
+        await retry.try(async () => {
+          await testSubjects.existOrFail('apmMainContainer', {
+            timeout: 10000,
+          });
+
+          const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer');
+          const apmMainContainerTextItems = apmMainContainerText[0].split('\n');
+          expect(apmMainContainerTextItems).to.contain('No services found');
+        });
+      });
+
+      it('sets the timePicker to return data', async () => {
+        await PageObjects.timePicker.timePickerExists();
+
+        const fromTime = 'Jul 29, 2019 @ 00:00:00.000';
+        const toTime = 'Jul 30, 2019 @ 00:00:00.000';
+        await PageObjects.timePicker.setAbsoluteRange(fromTime, toTime);
+
+        await retry.try(async () => {
+          const apmMainContainerText = await testSubjects.getVisibleTextAll('apmMainContainer');
+          const apmMainContainerTextItems = apmMainContainerText[0].split('\n');
+
+          expect(apmMainContainerTextItems).to.not.contain('No services found');
+
+          expect(apmMainContainerTextItems).to.contain('opbeans-go');
+          expect(apmMainContainerTextItems).to.contain('opbeans-node');
+          expect(apmMainContainerTextItems).to.contain('opbeans-ruby');
+          expect(apmMainContainerTextItems).to.contain('opbeans-python');
+          expect(apmMainContainerTextItems).to.contain('opbeans-dotnet');
+          expect(apmMainContainerTextItems).to.contain('opbeans-java');
+
+          expect(apmMainContainerTextItems).to.contain('development');
+
+          const items = await testSubjects.findAll('apmServiceListAppLink');
+          expect(items.length).to.be(6);
+        });
+      });
+
+      it(`navigates to the 'opbeans-go' service overview page`, async function () {
+        await find.clickByDisplayedLinkText(testData.serviceName);
+
+        await retry.try(async () => {
+          const apmMainTemplateHeaderServiceName = await testSubjects.getVisibleTextAll(
+            'apmMainTemplateHeaderServiceName'
+          );
+          expect(apmMainTemplateHeaderServiceName).to.contain('opbeans-go');
+        });
+      });
+
+      it('shows the correlations flyout', async function () {
+        await testSubjects.click('apmViewCorrelationsButton');
+
+        await retry.try(async () => {
+          await testSubjects.existOrFail('apmCorrelationsFlyout', {
+            timeout: 10000,
+          });
+
+          const apmCorrelationsFlyoutHeader = await testSubjects.getVisibleText(
+            'apmCorrelationsFlyoutHeader'
+          );
+
+          expect(apmCorrelationsFlyoutHeader).to.contain('Correlations BETA');
+        });
+      });
+
+      it('loads the correlation results', async function () {
+        await retry.try(async () => {
+          // Assert that the data fully loaded to 100%
+          const apmCorrelationsLatencyCorrelationsProgressTitle = await testSubjects.getVisibleText(
+            'apmCorrelationsLatencyCorrelationsProgressTitle'
+          );
+          expect(apmCorrelationsLatencyCorrelationsProgressTitle).to.be('Progress: 100%');
+
+          // Assert that the Correlations Chart and its header are present
+          const apmCorrelationsLatencyCorrelationsChartTitle = await testSubjects.getVisibleText(
+            'apmCorrelationsLatencyCorrelationsChartTitle'
+          );
+          expect(apmCorrelationsLatencyCorrelationsChartTitle).to.be(
+            `Latency distribution for ${testData.serviceName}`
+          );
+          await testSubjects.existOrFail('apmCorrelationsChart', {
+            timeout: 10000,
+          });
+
+          // Assert that results for the given service didn't find any correlations
+          const apmCorrelationsTable = await testSubjects.getVisibleText('apmCorrelationsTable');
+          expect(apmCorrelationsTable).to.be('No significant correlations found');
+        });
+      });
+    });
+  });
+}

x-pack/test/functional/apps/apm/index.ts

@@ -11,5 +11,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
   describe('APM specs', function () {
     this.tags('ciGroup6');
     loadTestFile(require.resolve('./feature_controls'));
+    loadTestFile(require.resolve('./correlations'));
   });
 }