[APM] Add warning to notify user about legacy ML jobs (#71030)

x-pack/plugins/apm/common/utils/range_filter.ts

@@ -4,13 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export function rangeFilter(
-  start: number,
-  end: number,
-  timestampField = '@timestamp'
-) {
+export function rangeFilter(start: number, end: number) {
   return {
-    [timestampField]: {
+    '@timestamp': {
       gte: start,
       lte: end,
       format: 'epoch_millis',

x-pack/plugins/apm/public/components/app/Settings/AgentConfigurations/index.tsx

@@ -25,7 +25,7 @@ export function AgentConfigurations() {
     (callApmApi) =>
       callApmApi({ pathname: '/api/apm/settings/agent-configuration' }),
     [],
-    { preservePreviousData: false }
+    { preservePreviousData: false, showToastOnError: false }
   );
 
   useTrackPageview({ app: 'apm', path: 'agent_configuration' });

x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/index.tsx

@@ -10,9 +10,15 @@ import { i18n } from '@kbn/i18n';
 import { EuiPanel } from '@elastic/eui';
 import { JobsList } from './jobs_list';
 import { AddEnvironments } from './add_environments';
-import { useFetcher, FETCH_STATUS } from '../../../../hooks/useFetcher';
+import { useFetcher } from '../../../../hooks/useFetcher';
 import { LicensePrompt } from '../../../shared/LicensePrompt';
 import { useLicense } from '../../../../hooks/useLicense';
+import { APIReturnType } from '../../../../services/rest/createCallApmApi';
+
+const DEFAULT_VALUE: APIReturnType<'/api/apm/settings/anomaly-detection'> = {
+  jobs: [],
+  hasLegacyJobs: false,
+};
 
 export const AnomalyDetection = () => {
   const license = useLicense();
@@ -20,17 +26,13 @@ export const AnomalyDetection = () => {
 
   const [viewAddEnvironments, setViewAddEnvironments] = useState(false);
 
-  const { refetch, data = [], status } = useFetcher(
+  const { refetch, data = DEFAULT_VALUE, status } = useFetcher(
     (callApmApi) =>
       callApmApi({ pathname: `/api/apm/settings/anomaly-detection` }),
     [],
-    { preservePreviousData: false }
+    { preservePreviousData: false, showToastOnError: false }
   );
 
-  const isLoading =
-    status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING;
-  const hasFetchFailure = status === FETCH_STATUS.FAILURE;
-
   if (!hasValidLicense) {
     return (
       <EuiPanel>
@@ -66,7 +68,7 @@ export const AnomalyDetection = () => {
       <EuiSpacer size="l" />
       {viewAddEnvironments ? (
         <AddEnvironments
-          currentEnvironments={data.map(({ environment }) => environment)}
+          currentEnvironments={data.jobs.map(({ environment }) => environment)}
           onCreateJobSuccess={() => {
             refetch();
             setViewAddEnvironments(false);
@@ -77,9 +79,9 @@ export const AnomalyDetection = () => {
         />
       ) : (
         <JobsList
-          isLoading={isLoading}
-          hasFetchFailure={hasFetchFailure}
-          anomalyDetectionJobsByEnv={data}
+          status={status}
+          anomalyDetectionJobsByEnv={data.jobs}
+          hasLegacyJobs={data.hasLegacyJobs}
           onAddEnvironments={() => {
             setViewAddEnvironments(true);
           }}

x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/jobs_list.tsx

@@ -16,12 +16,14 @@ import {
 } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
 import { FormattedMessage } from '@kbn/i18n/react';
+import { FETCH_STATUS } from '../../../../hooks/useFetcher';
 import { ITableColumn, ManagedTable } from '../../../shared/ManagedTable';
 import { LoadingStatePrompt } from '../../../shared/LoadingStatePrompt';
 import { AnomalyDetectionJobByEnv } from '../../../../../typings/anomaly_detection';
 import { MLJobLink } from '../../../shared/Links/MachineLearningLinks/MLJobLink';
 import { MLLink } from '../../../shared/Links/MachineLearningLinks/MLLink';
 import { ENVIRONMENT_NOT_DEFINED } from '../../../../../common/environment_filter_values';
+import { LegacyJobsCallout } from './legacy_jobs_callout';
 
 const columns: Array<ITableColumn<AnomalyDetectionJobByEnv>> = [
   {
@@ -60,17 +62,22 @@ const columns: Array<ITableColumn<AnomalyDetectionJobByEnv>> = [
 ];
 
 interface Props {
-  isLoading: boolean;
-  hasFetchFailure: boolean;
+  status: FETCH_STATUS;
   onAddEnvironments: () => void;
   anomalyDetectionJobsByEnv: AnomalyDetectionJobByEnv[];
+  hasLegacyJobs: boolean;
 }
 export const JobsList = ({
-  isLoading,
-  hasFetchFailure,
+  status,
   onAddEnvironments,
   anomalyDetectionJobsByEnv,
+  hasLegacyJobs,
 }: Props) => {
+  const isLoading =
+    status === FETCH_STATUS.PENDING || status === FETCH_STATUS.LOADING;
+
+  const hasFetchFailure = status === FETCH_STATUS.FAILURE;
+
   return (
     <EuiPanel>
       <EuiFlexGroup>
@@ -131,6 +138,8 @@ export const JobsList = ({
         items={isLoading || hasFetchFailure ? [] : anomalyDetectionJobsByEnv}
       />
       <EuiSpacer size="l" />
+
+      {hasLegacyJobs && <LegacyJobsCallout />}
     </EuiPanel>
   );
 };

x-pack/plugins/apm/public/components/app/Settings/anomaly_detection/legacy_jobs_callout.tsx

@@ -0,0 +1,43 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { EuiCallOut, EuiButton } from '@elastic/eui';
+import React from 'react';
+import { i18n } from '@kbn/i18n';
+import { useApmPluginContext } from '../../../../hooks/useApmPluginContext';
+
+export function LegacyJobsCallout() {
+  const { core } = useApmPluginContext();
+  return (
+    <EuiCallOut
+      title={i18n.translate(
+        'xpack.apm.settings.anomaly_detection.legacy_jobs.title',
+        { defaultMessage: 'Legacy ML jobs are no longer used in APM app' }
+      )}
+      iconType="iInCircle"
+    >
+      <p>
+        {i18n.translate(
+          'xpack.apm.settings.anomaly_detection.legacy_jobs.body',
+          {
+            defaultMessage:
+              'We have discovered legacy Machine Learning jobs from our previous integration which are no longer being used in the APM app',
+          }
+        )}
+      </p>
+      <EuiButton
+        href={core.http.basePath.prepend(
+          '/app/ml#/jobs?mlManagement=(jobId:high_mean_response_time)'
+        )}
+      >
+        {i18n.translate(
+          'xpack.apm.settings.anomaly_detection.legacy_jobs.button',
+          { defaultMessage: 'Review jobs' }
+        )}
+      </EuiButton>
+    </EuiCallOut>
+  );
+}

x-pack/plugins/apm/server/lib/anomaly_detection/constants.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export const ML_MODULE_ID_APM_TRANSACTION = 'apm_transaction';
+export const APM_ML_JOB_GROUP = 'apm';

x-pack/plugins/apm/server/lib/anomaly_detection/create_anomaly_detection_jobs.ts

@@ -14,9 +14,7 @@ import {
   PROCESSOR_EVENT,
 } from '../../../common/elasticsearch_fieldnames';
 import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values';
-
-const ML_MODULE_ID_APM_TRANSACTION = 'apm_transaction';
-export const ML_GROUP_NAME_APM = 'apm';
+import { APM_ML_JOB_GROUP, ML_MODULE_ID_APM_TRANSACTION } from './constants';
 
 export type CreateAnomalyDetectionJobsAPIResponse = PromiseReturnType<
   typeof createAnomalyDetectionJobs
@@ -83,8 +81,8 @@ async function createAnomalyDetectionJob({
 
   return ml.modules.setup({
     moduleId: ML_MODULE_ID_APM_TRANSACTION,
-    prefix: `${ML_GROUP_NAME_APM}-${convertedEnvironmentName}-${randomToken}-`,
-    groups: [ML_GROUP_NAME_APM, convertedEnvironmentName],
+    prefix: `${APM_ML_JOB_GROUP}-${convertedEnvironmentName}-${randomToken}-`,
+    groups: [APM_ML_JOB_GROUP, convertedEnvironmentName],
     indexPatternName,
     query: {
       bool: {

x-pack/plugins/apm/server/lib/anomaly_detection/get_anomaly_detection_jobs.ts

@@ -5,56 +5,34 @@
  */
 
 import { Logger } from 'kibana/server';
-import { PromiseReturnType } from '../../../../observability/typings/common';
 import { Setup } from '../helpers/setup_request';
-import { AnomalyDetectionJobByEnv } from '../../../typings/anomaly_detection';
-import { ML_GROUP_NAME_APM } from './create_anomaly_detection_jobs';
+import { getMlJobsWithAPMGroup } from './get_ml_jobs_by_group';
 
-export type AnomalyDetectionJobsAPIResponse = PromiseReturnType<
-  typeof getAnomalyDetectionJobs
->;
-export async function getAnomalyDetectionJobs(
-  setup: Setup,
-  logger: Logger
-): Promise<AnomalyDetectionJobByEnv[]> {
+export async function getAnomalyDetectionJobs(setup: Setup, logger: Logger) {
   const { ml } = setup;
   if (!ml) {
     return [];
   }
-  try {
-    const mlCapabilities = await ml.mlSystem.mlCapabilities();
-    if (
-      !(
-        mlCapabilities.mlFeatureEnabledInSpace &&
-        mlCapabilities.isPlatinumOrTrialLicense
-      )
-    ) {
-      logger.warn(
-        'Anomaly detection integration is not availble for this user.'
-      );
-      return [];
-    }
-  } catch (error) {
-    logger.warn('Unable to get ML capabilities.');
-    logger.error(error);
-    return [];
-  }
-  try {
-    const { jobs } = await ml.anomalyDetectors.jobs(ML_GROUP_NAME_APM);
-    return jobs
-      .map((job) => {
-        const environment = job.custom_settings?.job_tags?.environment ?? '';
-        return {
-          job_id: job.job_id,
-          environment,
-        };
-      })
-      .filter((job) => job.environment);
-  } catch (error) {
-    if (error.statusCode !== 404) {
-      logger.warn('Unable to get APM ML jobs.');
-      logger.error(error);
-    }
+
+  const mlCapabilities = await ml.mlSystem.mlCapabilities();
+  if (
+    !(
+      mlCapabilities.mlFeatureEnabledInSpace &&
+      mlCapabilities.isPlatinumOrTrialLicense
+    )
+  ) {
+    logger.warn('Anomaly detection integration is not availble for this user.');
     return [];
   }
+
+  const response = await getMlJobsWithAPMGroup(ml);
+  return response.jobs
+    .map((job) => {
+      const environment = job.custom_settings?.job_tags?.environment ?? '';
+      return {
+        job_id: job.job_id,
+        environment,
+      };
+    })
+    .filter((job) => job.environment);
 }

x-pack/plugins/apm/server/lib/anomaly_detection/get_ml_jobs_by_group.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Setup } from '../helpers/setup_request';
+import { APM_ML_JOB_GROUP } from './constants';
+
+// returns ml jobs containing "apm" group
+// workaround: the ML api returns 404 when no jobs are found. This is handled so instead of throwing an empty response is returned
+export async function getMlJobsWithAPMGroup(ml: NonNullable<Setup['ml']>) {
+  try {
+    return await ml.anomalyDetectors.jobs(APM_ML_JOB_GROUP);
+  } catch (e) {
+    if (e.statusCode === 404) {
+      return { count: 0, jobs: [] };
+    }
+
+    throw e;
+  }
+}

x-pack/plugins/apm/server/lib/anomaly_detection/has_legacy_jobs.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { Setup } from '../helpers/setup_request';
+import { getMlJobsWithAPMGroup } from './get_ml_jobs_by_group';
+
+// Determine whether there are any legacy ml jobs.
+// A legacy ML job has a job id that ends with "high_mean_response_time" and created_by=ml-module-apm-transaction
+export async function hasLegacyJobs(setup: Setup) {
+  const { ml } = setup;
+
+  if (!ml) {
+    return false;
+  }
+
+  const response = await getMlJobsWithAPMGroup(ml);
+  return response.jobs.some(
+    (job) =>
+      job.job_id.endsWith('high_mean_response_time') &&
+      job.custom_settings?.created_by === 'ml-module-apm-transaction'
+  );
+}

x-pack/plugins/apm/server/routes/settings/anomaly_detection.ts

@@ -10,6 +10,7 @@ import { getAnomalyDetectionJobs } from '../../lib/anomaly_detection/get_anomaly
 import { createAnomalyDetectionJobs } from '../../lib/anomaly_detection/create_anomaly_detection_jobs';
 import { setupRequest } from '../../lib/helpers/setup_request';
 import { getAllEnvironments } from '../../lib/environments/get_all_environments';
+import { hasLegacyJobs } from '../../lib/anomaly_detection/has_legacy_jobs';
 
 // get ML anomaly detection jobs for each environment
 export const anomalyDetectionJobsRoute = createRoute(() => ({
@@ -17,7 +18,11 @@ export const anomalyDetectionJobsRoute = createRoute(() => ({
   path: '/api/apm/settings/anomaly-detection',
   handler: async ({ context, request }) => {
     const setup = await setupRequest(context, request);
-    return await getAnomalyDetectionJobs(setup, context.logger);
+    const jobs = await getAnomalyDetectionJobs(setup, context.logger);
+    return {
+      jobs,
+      hasLegacyJobs: await hasLegacyJobs(setup),
+    };
   },
 }));
 

x-pack/plugins/ml/common/constants/new_job.ts

@@ -17,6 +17,7 @@ export enum CREATED_BY_LABEL {
   MULTI_METRIC = 'multi-metric-wizard',
   POPULATION = 'population-wizard',
   CATEGORIZATION = 'categorization-wizard',
+  APM_TRANSACTION = 'ml-module-apm-transaction',
 }
 
 export const DEFAULT_MODEL_MEMORY_LIMIT = '10MB';