[ML] Fixing cloud behaviour when ML nodes are lazily assigned (#90051)

* [ML] Fixing cloud behaviour when ML nodes are lazily assigned

* fixing duplicate id

* updating snapshot

* removing comments

* fixing tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts

@@ -32,6 +32,7 @@ export interface MlSummaryJob {
   deleting?: boolean;
   latestTimestampSortValue?: number;
   earliestStartTimestampMs?: number;
+  awaitingNodeAssignment: boolean;
 }
 
 export interface AuditMessage {

x-pack/plugins/ml/common/types/ml_server_info.ts

@@ -31,3 +31,8 @@ export interface MlInfoResponse {
   upgrade_mode: boolean;
   cloudId?: string;
 }
+
+export interface MlNodeCount {
+  count: number;
+  lazyNodeCount: number;
+}

x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { JobsAwaitingNodeWarning } from './jobs_awaiting_node_warning';
+export { NewJobAwaitingNodeWarning } from './new_job_awaiting_node';

x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/jobs_awaiting_node_warning.tsx

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { Fragment, FC } from 'react';
+
+import { EuiCallOut, EuiSpacer } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { isCloud } from '../../services/ml_server_info';
+
+interface Props {
+  jobCount: number;
+}
+
+export const JobsAwaitingNodeWarning: FC<Props> = ({ jobCount }) => {
+  if (isCloud() === false || jobCount === 0) {
+    return null;
+  }
+
+  return (
+    <Fragment>
+      <EuiCallOut
+        title={
+          <FormattedMessage
+            id="xpack.ml.jobsAwaitingNodeWarning.title"
+            defaultMessage="Awaiting ML node provisioning"
+          />
+        }
+        color="primary"
+        iconType="iInCircle"
+      >
+        <div>
+          <FormattedMessage
+            id="xpack.ml.jobsAwaitingNodeWarning.noMLNodesAvailableDescription"
+            defaultMessage="There {jobCount, plural, one {is} other {are}} {jobCount, plural, one {# job} other {# jobs}} waiting to be started while ML nodes are being provisioned."
+            values={{
+              jobCount,
+            }}
+          />
+        </div>
+      </EuiCallOut>
+      <EuiSpacer size="m" />
+    </Fragment>
+  );
+};

x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/new_job_awaiting_node.tsx

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { Fragment, FC } from 'react';
+
+import { EuiCallOut, EuiSpacer } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+import { JobType } from '../../../../common/types/saved_objects';
+
+interface Props {
+  jobType: JobType;
+}
+
+export const NewJobAwaitingNodeWarning: FC<Props> = () => {
+  return (
+    <Fragment>
+      <EuiCallOut
+        title={
+          <FormattedMessage
+            id="xpack.ml.jobsAwaitingNodeWarning.title"
+            defaultMessage="Awaiting ML node provisioning"
+          />
+        }
+        color="primary"
+        iconType="iInCircle"
+      >
+        <div>
+          <FormattedMessage
+            id="xpack.ml.newJobAwaitingNodeWarning.noMLNodesAvailableDescription"
+            defaultMessage="Job cannot be started straight away, an ML node needs to be started. This will happen automatically."
+          />
+        </div>
+      </EuiCallOut>
+      <EuiSpacer size="m" />
+    </Fragment>
+  );
+};

x-pack/plugins/ml/public/application/components/node_available_warning/node_available_warning.tsx

@@ -41,7 +41,7 @@ export const NodeAvailableWarning: FC = () => {
             defaultMessage="You will not be able to create or run jobs."
           />
         </div>
-        {isCloud && id !== null && (
+        {isCloud() && id !== null && (
           <div>
             <FormattedMessage
               id="xpack.ml.jobsList.nodeAvailableWarning.linkToCloudDescription"

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/create_step_footer/create_step_footer.tsx

@@ -19,6 +19,7 @@ import { BackToListPanel } from '../back_to_list_panel';
 import { ViewResultsPanel } from '../view_results_panel';
 import { ProgressStats } from './progress_stats';
 import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
+import { NewJobAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
 
 export const PROGRESS_REFRESH_INTERVAL_MS = 1000;
 
@@ -41,6 +42,7 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
   const [currentProgress, setCurrentProgress] = useState<AnalyticsProgressStats | undefined>(
     undefined
   );
+  const [showJobAssignWarning, setShowJobAssignWarning] = useState(false);
 
   const {
     services: { notifications },
@@ -58,6 +60,10 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
           ? analyticsStats.data_frame_analytics[0]
           : undefined;
 
+        setShowJobAssignWarning(
+          jobStats?.state === DATA_FRAME_TASK_STATE.STARTING && jobStats?.node === undefined
+        );
+
         if (jobStats !== undefined) {
           const progressStats = getDataFrameAnalyticsProgressPhase(jobStats);
 
@@ -106,25 +112,28 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
   }, [initialized]);
 
   return (
-    <EuiFlexGroup direction="column">
+    <>
-      <EuiFlexItem grow={false}>
+      {showJobAssignWarning && <NewJobAwaitingNodeWarning jobType="data-frame-analytics" />}
-        {showProgress && (
+      <EuiFlexGroup direction="column">
-          <ProgressStats currentProgress={currentProgress} failedJobMessage={failedJobMessage} />
+        <EuiFlexItem grow={false}>
-        )}
+          {showProgress && (
-      </EuiFlexItem>
+            <ProgressStats currentProgress={currentProgress} failedJobMessage={failedJobMessage} />
-      <EuiFlexItem grow={false}>
-        <EuiHorizontalRule />
-        <EuiFlexGroup>
-          <EuiFlexItem grow={false}>
-            <BackToListPanel />
-          </EuiFlexItem>
-          {jobFinished === true && (
-            <EuiFlexItem grow={false}>
-              <ViewResultsPanel jobId={jobId} analysisType={jobType} />
-            </EuiFlexItem>
           )}
-        </EuiFlexGroup>
+        </EuiFlexItem>
-      </EuiFlexItem>
+        <EuiFlexItem grow={false}>
-    </EuiFlexGroup>
+          <EuiHorizontalRule />
+          <EuiFlexGroup>
+            <EuiFlexItem grow={false}>
+              <BackToListPanel />
+            </EuiFlexItem>
+            {jobFinished === true && (
+              <EuiFlexItem grow={false}>
+                <ViewResultsPanel jobId={jobId} analysisType={jobType} />
+              </EuiFlexItem>
+            )}
+          </EuiFlexGroup>
+        </EuiFlexItem>
+      </EuiFlexGroup>
+    </>
   );
 };

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx

@@ -36,6 +36,7 @@ import { AnalyticsEmptyPrompt } from './empty_prompt';
 import { useTableSettings } from './use_table_settings';
 import { RefreshAnalyticsListButton } from '../refresh_analytics_list_button';
 import { ListingPageUrlState } from '../../../../../../../common/types/common';
+import { JobsAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
 
 const filters: EuiSearchBarProps['filters'] = [
   {
@@ -114,6 +115,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
   );
   const [expandedRowItemIds, setExpandedRowItemIds] = useState<DataFrameAnalyticsId[]>([]);
   const [errorMessage, setErrorMessage] = useState<any>(undefined);
+  const [jobsAwaitingNodeCount, setJobsAwaitingNodeCount] = useState(0);
 
   const disabled =
     !checkPermission('canCreateDataFrameAnalytics') ||
@@ -124,6 +126,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
     setAnalyticsStats,
     setErrorMessage,
     setIsInitialized,
+    setJobsAwaitingNodeCount,
     blockRefresh,
     isManagementTable
   );
@@ -261,6 +264,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
     <div data-test-subj="mlAnalyticsJobList">
       {modals}
       {!isManagementTable && <EuiSpacer size="m" />}
+      <JobsAwaitingNodeWarning jobCount={jobsAwaitingNodeCount} />
       <EuiFlexGroup justifyContent="spaceBetween">
         {!isManagementTable && stats}
         {isManagementTable && managementStats}

x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/get_analytics.ts

@@ -26,6 +26,7 @@ import {
 } from '../../components/analytics_list/common';
 import { AnalyticStatsBarStats } from '../../../../../components/stats_bar';
 import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
+import { DATA_FRAME_TASK_STATE } from '../../../../../../../common/constants/data_frame_analytics';
 
 export const isGetDataFrameAnalyticsStatsResponseOk = (
   arg: any
@@ -106,6 +107,7 @@ export const getAnalyticsFactory = (
     React.SetStateAction<GetDataFrameAnalyticsStatsResponseError | undefined>
   >,
   setIsInitialized: React.Dispatch<React.SetStateAction<boolean>>,
+  setJobsAwaitingNodeCount: React.Dispatch<React.SetStateAction<number>>,
   blockRefresh: boolean,
   isManagementTable: boolean
 ): GetAnalytics => {
@@ -134,6 +136,8 @@ export const getAnalyticsFactory = (
           ? getAnalyticsJobsStats(analyticsStats)
           : undefined;
 
+        let jobsAwaitingNodeCount = 0;
+
         const tableRows = analyticsConfigs.data_frame_analytics.reduce(
           (reducedtableRows, config) => {
             const stats = isGetDataFrameAnalyticsStatsResponseOk(analyticsStats)
@@ -146,6 +150,10 @@ export const getAnalyticsFactory = (
               return reducedtableRows;
             }
 
+            if (stats.state === DATA_FRAME_TASK_STATE.STARTING && stats.node === undefined) {
+              jobsAwaitingNodeCount++;
+            }
+
             // Table with expandable rows requires `id` on the outer most level
             reducedtableRows.push({
               checkpointing: {},
@@ -166,6 +174,7 @@ export const getAnalyticsFactory = (
         setAnalyticsStats(analyticsStatsResult);
         setErrorMessage(undefined);
         setIsInitialized(true);
+        setJobsAwaitingNodeCount(jobsAwaitingNodeCount);
         refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE);
       } catch (e) {
         // An error is followed immediately by setting the state to idle.

x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js

@@ -32,6 +32,7 @@ import { MultiJobActions } from '../multi_job_actions';
 import { NewJobButton } from '../new_job_button';
 import { JobStatsBar } from '../jobs_stats_bar';
 import { NodeAvailableWarning } from '../../../../components/node_available_warning';
+import { JobsAwaitingNodeWarning } from '../../../../components/jobs_awaiting_node_warning';
 import { SavedObjectsWarning } from '../../../../components/saved_objects_warning';
 import { DatePickerWrapper } from '../../../../components/navigation_menu/date_picker_wrapper';
 import { UpgradeWarning } from '../../../../components/upgrade';
@@ -56,6 +57,7 @@ export class JobsListView extends Component {
       itemIdToExpandedRowMap: {},
       filterClauses: [],
       deletingJobIds: [],
+      jobsAwaitingNodeCount: 0,
     };
 
     this.spacesEnabled = props.spacesEnabled ?? false;
@@ -272,6 +274,7 @@ export class JobsListView extends Component {
           spaces = allSpaces['anomaly-detector'];
         }
 
+        let jobsAwaitingNodeCount = 0;
         const jobs = await ml.jobs.jobsSummary(expandedJobsIds);
         const fullJobsList = {};
         const jobsSummaryList = jobs.map((job) => {
@@ -287,11 +290,21 @@ export class JobsListView extends Component {
             spaces[job.id] !== undefined
               ? spaces[job.id]
               : [];
+
+          if (job.awaitingNodeAssignment === true) {
+            jobsAwaitingNodeCount++;
+          }
           return job;
         });
         const filteredJobsSummaryList = filterJobs(jobsSummaryList, this.state.filterClauses);
         this.setState(
-          { jobsSummaryList, filteredJobsSummaryList, fullJobsList, loading: false },
+          {
+            jobsSummaryList,
+            filteredJobsSummaryList,
+            fullJobsList,
+            loading: false,
+            jobsAwaitingNodeCount,
+          },
           () => {
             this.refreshSelectedJobs();
           }
@@ -407,7 +420,7 @@ export class JobsListView extends Component {
   }
 
   renderJobsListComponents() {
-    const { isRefreshing, loading, jobsSummaryList } = this.state;
+    const { isRefreshing, loading, jobsSummaryList, jobsAwaitingNodeCount } = this.state;
     const jobIds = jobsSummaryList.map((j) => j.id);
 
     return (
@@ -440,6 +453,7 @@ export class JobsListView extends Component {
           </EuiPageHeader>
 
           <NodeAvailableWarning />
+          <JobsAwaitingNodeWarning jobCount={jobsAwaitingNodeCount} />
           <SavedObjectsWarning jobType="anomaly-detector" />
 
           <UpgradeWarning />

x-pack/plugins/ml/public/application/jobs/new_job/common/job_runner/job_runner.ts

@@ -11,12 +11,14 @@ import { JobCreator } from '../job_creator';
 import { DatafeedId, JobId } from '../../../../../../common/types/anomaly_detection_jobs';
 import { DATAFEED_STATE } from '../../../../../../common/constants/states';
 
-const REFRESH_INTERVAL_MS = 100;
+const REFRESH_INTERVAL_MS = 250;
+const NODE_ASSIGNMENT_CHECK_REFRESH_INTERVAL_MS = 2000;
 const TARGET_PROGRESS_DELTA = 2;
 const REFRESH_RATE_ADJUSTMENT_DELAY_MS = 2000;
 
 type Progress = number;
 export type ProgressSubscriber = (progress: number) => void;
+export type JobAssignmentSubscriber = (assigned: boolean) => void;
 
 export class JobRunner {
   private _jobId: JobId;
@@ -35,6 +37,8 @@ export class JobRunner {
 
   private _datafeedStartTime: number = 0;
   private _performRefreshRateAdjustment: boolean = false;
+  private _jobAssignedToNode: boolean = false;
+  private _jobAssignedToNode$: BehaviorSubject<boolean>;
 
   constructor(jobCreator: JobCreator) {
     this._jobId = jobCreator.jobId;
@@ -45,6 +49,7 @@ export class JobRunner {
     this._stopRefreshPoll = jobCreator.stopAllRefreshPolls;
 
     this._progress$ = new BehaviorSubject(this._percentageComplete);
+    this._jobAssignedToNode$ = new BehaviorSubject(this._jobAssignedToNode);
     this._subscribers = jobCreator.subscribers;
   }
 
@@ -62,7 +67,9 @@ export class JobRunner {
 
   private async openJob(): Promise<void> {
     try {
-      await mlJobService.openJob(this._jobId);
+      const { node }: { node?: string } = await mlJobService.openJob(this._jobId);
+      this._jobAssignedToNode = node !== undefined && node.length > 0;
+      this._jobAssignedToNode$.next(this._jobAssignedToNode);
     } catch (error) {
       throw error;
     }
@@ -94,7 +101,7 @@ export class JobRunner {
       this._datafeedState = DATAFEED_STATE.STARTED;
       this._percentageComplete = 0;
 
-      const check = async () => {
+      const checkProgress = async () => {
         const { isRunning, progress: prog, isJobClosed } = await this.getProgress();
 
         // if the progress has reached 100% but the job is still running,
@@ -109,7 +116,9 @@ export class JobRunner {
 
         if ((isRunning === true || isJobClosed === false) && this._stopRefreshPoll.stop === false) {
           setTimeout(async () => {
-            await check();
+            if (this._stopRefreshPoll.stop === false) {
+              await checkProgress();
+            }
           }, this._refreshInterval);
         } else {
           // job has finished running, set progress to 100%
@@ -121,11 +130,30 @@ export class JobRunner {
           subscriptions.forEach((s) => s.unsubscribe());
         }
       };
+
+      const checkJobIsAssigned = async () => {
+        this._jobAssignedToNode = await this._isJobAssigned();
+        this._jobAssignedToNode$.next(this._jobAssignedToNode);
+        if (this._jobAssignedToNode === true) {
+          await checkProgress();
+        } else {
+          setTimeout(async () => {
+            if (this._stopRefreshPoll.stop === false) {
+              await checkJobIsAssigned();
+            }
+          }, NODE_ASSIGNMENT_CHECK_REFRESH_INTERVAL_MS);
+        }
+      };
       // wait for the first check to run and then return success.
       // all subsequent checks will update the observable
       if (pollProgress === true) {
-        await check();
+        if (this._jobAssignedToNode === true) {
+          await checkProgress();
+        } else {
+          await checkJobIsAssigned();
+        }
       }
+
       return started;
     } catch (error) {
       throw error;
@@ -159,6 +187,11 @@ export class JobRunner {
     }
   }
 
+  private async _isJobAssigned(): Promise<boolean> {
+    const { jobs } = await ml.getJobStats({ jobId: this._jobId });
+    return jobs.length > 0 && jobs[0].node !== undefined;
+  }
+
   public async startDatafeed() {
     return await this._startDatafeed(this._start, this._end, true);
   }
@@ -185,4 +218,12 @@ export class JobRunner {
     const { isRunning } = await this.getProgress();
     return isRunning;
   }
+
+  public isJobAssignedToNode() {
+    return this._jobAssignedToNode;
+  }
+
+  public subscribeToJobAssignment(func: JobAssignmentSubscriber) {
+    return this._jobAssignedToNode$.subscribe(func);
+  }
 }

x-pack/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/summary.tsx

@@ -5,6 +5,7 @@
  */
 
 import React, { Fragment, FC, useContext, useState, useEffect } from 'react';
+import { Subscription } from 'rxjs';
 import {
   EuiButton,
   EuiButtonEmpty,
@@ -29,6 +30,7 @@ import { DetectorChart } from './components/detector_chart';
 import { JobProgress } from './components/job_progress';
 import { PostSaveOptions } from './components/post_save_options';
 import { StartDatafeedSwitch } from './components/start_datafeed_switch';
+import { NewJobAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
 import { toastNotificationServiceProvider } from '../../../../../services/toast_notification_service';
 import {
   convertToAdvancedJob,
@@ -55,6 +57,7 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
   const [isValid, setIsValid] = useState(jobValidator.validationSummary.basic);
   const [jobRunner, setJobRunner] = useState<JobRunner | null>(null);
   const [startDatafeed, setStartDatafeed] = useState(true);
+  const [showJobAssignWarning, setShowJobAssignWarning] = useState(false);
 
   const isAdvanced = isAdvancedJobCreator(jobCreator);
   const jsonEditorMode = isAdvanced ? EDITOR_MODE.EDITABLE : EDITOR_MODE.READONLY;
@@ -63,6 +66,20 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
     jobCreator.subscribeToProgress(setProgress);
   }, []);
 
+  useEffect(() => {
+    let s: Subscription | null = null;
+    if (jobRunner !== null) {
+      s = jobRunner.subscribeToJobAssignment((assigned: boolean) =>
+        setShowJobAssignWarning(!assigned)
+      );
+    }
+    return () => {
+      if (s !== null) {
+        s?.unsubscribe();
+      }
+    };
+  }, [jobRunner]);
+
   async function start() {
     setCreatingJob(true);
     if (isAdvanced) {
@@ -155,6 +172,7 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
           )}
 
           <EuiHorizontalRule />
+          {showJobAssignWarning && <NewJobAwaitingNodeWarning jobType="anomaly-detector" />}
           <EuiFlexGroup>
             {progress < 100 && (
               <Fragment>

x-pack/plugins/ml/public/application/ml_nodes_check/check_ml_nodes.ts

@@ -5,14 +5,16 @@
  */
 
 import { ml } from '../services/ml_api_service';
+import { MlNodeCount } from '../../../common/types/ml_server_info';
 
 let mlNodeCount: number = 0;
+let lazyMlNodeCount: number = 0;
 let userHasPermissionToViewMlNodeCount: boolean = false;
 
 export async function checkMlNodesAvailable(redirectToJobsManagementPage: () => Promise<void>) {
   try {
-    const nodes = await getMlNodeCount();
+    const { count, lazyNodeCount } = await getMlNodeCount();
-    if (nodes.count !== undefined && nodes.count > 0) {
+    if (count > 0 || lazyNodeCount > 0) {
       Promise.resolve();
     } else {
       throw Error('Cannot load count of ML nodes');
@@ -25,23 +27,24 @@ export async function checkMlNodesAvailable(redirectToJobsManagementPage: () =>
   }
 }
 
-export async function getMlNodeCount() {
+export async function getMlNodeCount(): Promise<MlNodeCount> {
   try {
     const nodes = await ml.mlNodeCount();
     mlNodeCount = nodes.count;
+    lazyMlNodeCount = nodes.lazyNodeCount;
     userHasPermissionToViewMlNodeCount = true;
-    return Promise.resolve(nodes);
+    return nodes;
   } catch (error) {
     mlNodeCount = 0;
     if (error.statusCode === 403) {
       userHasPermissionToViewMlNodeCount = false;
     }
-    return Promise.resolve({ count: 0 });
+    return { count: 0, lazyNodeCount: 0 };
   }
 }
 
 export function mlNodesAvailable() {
-  return mlNodeCount !== 0;
+  return mlNodeCount !== 0 || lazyMlNodeCount !== 0;
 }
 
 export function permissionToViewMlNodeCount() {

x-pack/plugins/ml/public/application/overview/components/analytics_panel/analytics_panel.tsx

@@ -28,8 +28,9 @@ import { ML_PAGES } from '../../../../../common/constants/ml_url_generator';
 
 interface Props {
   jobCreationDisabled: boolean;
+  setLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
 }
-export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled }) => {
+export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled, setLazyJobCount }) => {
   const [analytics, setAnalytics] = useState<DataFrameAnalyticsListRow[]>([]);
   const [analyticsStats, setAnalyticsStats] = useState<AnalyticStatsBarStats | undefined>(
     undefined
@@ -52,6 +53,7 @@ export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled }) => {
     setAnalyticsStats,
     setErrorMessage,
     setIsInitialized,
+    setLazyJobCount,
     false,
     false
   );

x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/anomaly_detection_panel.tsx

@@ -51,9 +51,10 @@ function getDefaultAnomalyScores(groups: Group[]): MaxScoresByGroup {
 
 interface Props {
   jobCreationDisabled: boolean;
+  setLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
 }
 
-export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
+export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled, setLazyJobCount }) => {
   const {
     services: { notifications },
   } = useMlKibana();
@@ -84,10 +85,14 @@ export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
   const loadJobs = async () => {
     setIsLoading(true);
 
+    let lazyJobCount = 0;
     try {
       const jobsResult: MlSummaryJobs = await ml.jobs.jobsSummary([]);
       const jobsSummaryList = jobsResult.map((job: MlSummaryJob) => {
         job.latestTimestampSortValue = job.latestTimestampMs || 0;
+        if (job.awaitingNodeAssignment) {
+          lazyJobCount++;
+        }
         return job;
       });
       const { groups: jobsGroups, count } = getGroupsFromJobs(jobsSummaryList);
@@ -100,6 +105,7 @@ export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
       setGroups(jobsGroups);
       setJobsList(jobsWithTimerange);
       loadMaxAnomalyScores(jobsGroups);
+      setLazyJobCount(lazyJobCount);
     } catch (e) {
       setErrorMessage(e.message !== undefined ? e.message : JSON.stringify(e));
       setIsLoading(false);

x-pack/plugins/ml/public/application/overview/components/content.tsx

@@ -12,20 +12,30 @@ import { AnalyticsPanel } from './analytics_panel';
 interface Props {
   createAnomalyDetectionJobDisabled: boolean;
   createAnalyticsJobDisabled: boolean;
+  setAdLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
+  setDfaLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
 }
 
 // Fetch jobs and determine what to show
 export const OverviewContent: FC<Props> = ({
   createAnomalyDetectionJobDisabled,
   createAnalyticsJobDisabled,
+  setAdLazyJobCount,
+  setDfaLazyJobCount,
 }) => (
   <EuiFlexItem grow={3}>
     <EuiFlexGroup direction="column">
       <EuiFlexItem grow={false}>
-        <AnomalyDetectionPanel jobCreationDisabled={createAnomalyDetectionJobDisabled} />
+        <AnomalyDetectionPanel
+          jobCreationDisabled={createAnomalyDetectionJobDisabled}
+          setLazyJobCount={setAdLazyJobCount}
+        />
       </EuiFlexItem>
       <EuiFlexItem grow={false}>
-        <AnalyticsPanel jobCreationDisabled={createAnalyticsJobDisabled} />
+        <AnalyticsPanel
+          jobCreationDisabled={createAnalyticsJobDisabled}
+          setLazyJobCount={setDfaLazyJobCount}
+        />
       </EuiFlexItem>
     </EuiFlexGroup>
   </EuiFlexItem>

x-pack/plugins/ml/public/application/overview/overview_page.tsx

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import React, { Fragment, FC } from 'react';
+import React, { Fragment, FC, useState } from 'react';
 import { EuiFlexGroup, EuiPage, EuiPageBody } from '@elastic/eui';
 import { checkPermission } from '../capabilities/check_capabilities';
 import { mlNodesAvailable } from '../ml_nodes_check/check_ml_nodes';
@@ -12,6 +12,7 @@ import { NavigationMenu } from '../components/navigation_menu';
 import { OverviewSideBar } from './components/sidebar';
 import { OverviewContent } from './components/content';
 import { NodeAvailableWarning } from '../components/node_available_warning';
+import { JobsAwaitingNodeWarning } from '../components/jobs_awaiting_node_warning';
 import { SavedObjectsWarning } from '../components/saved_objects_warning';
 import { UpgradeWarning } from '../components/upgrade';
 import { HelpMenu } from '../components/help_menu';
@@ -27,12 +28,17 @@ export const OverviewPage: FC = () => {
     services: { docLinks },
   } = useMlKibana();
   const helpLink = docLinks.links.ml.guide;
+
+  const [adLazyJobCount, setAdLazyJobCount] = useState(0);
+  const [dfaLazyJobCount, setDfaLazyJobCount] = useState(0);
+
   return (
     <Fragment>
       <NavigationMenu tabId="overview" />
       <EuiPage data-test-subj="mlPageOverview">
         <EuiPageBody>
           <NodeAvailableWarning />
+          <JobsAwaitingNodeWarning jobCount={adLazyJobCount + dfaLazyJobCount} />
           <SavedObjectsWarning />
           <UpgradeWarning />
 
@@ -41,6 +47,8 @@ export const OverviewPage: FC = () => {
             <OverviewContent
               createAnomalyDetectionJobDisabled={disableCreateAnomalyDetectionJob}
               createAnalyticsJobDisabled={disableCreateAnalyticsButton}
+              setAdLazyJobCount={setAdLazyJobCount}
+              setDfaLazyJobCount={setDfaLazyJobCount}
             />
           </EuiFlexGroup>
         </EuiPageBody>

x-pack/plugins/ml/public/application/services/ml_api_service/index.ts

@@ -15,12 +15,17 @@ import { resultsApiProvider } from './results';
 import { jobsApiProvider } from './jobs';
 import { fileDatavisualizer } from './datavisualizer';
 import { savedObjectsApiProvider } from './saved_objects';
-import { MlServerDefaults, MlServerLimits } from '../../../../common/types/ml_server_info';
+import {
+  MlServerDefaults,
+  MlServerLimits,
+  MlNodeCount,
+} from '../../../../common/types/ml_server_info';
 
 import { MlCapabilitiesResponse } from '../../../../common/types/capabilities';
 import { Calendar, CalendarId, UpdateCalendar } from '../../../../common/types/calendars';
 import {
   Job,
+  JobStats,
   Datafeed,
   CombinedJob,
   Detector,
@@ -116,14 +121,14 @@ export function mlApiServicesProvider(httpService: HttpService) {
   return {
     getJobs(obj?: { jobId?: string }) {
       const jobId = obj && obj.jobId ? `/${obj.jobId}` : '';
-      return httpService.http<any>({
+      return httpService.http<{ jobs: Job[]; count: number }>({
         path: `${basePath()}/anomaly_detectors${jobId}`,
       });
     },
 
     getJobStats(obj: { jobId?: string }) {
       const jobId = obj && obj.jobId ? `/${obj.jobId}` : '';
-      return httpService.http<any>({
+      return httpService.http<{ jobs: JobStats[]; count: number }>({
         path: `${basePath()}/anomaly_detectors${jobId}/_stats`,
       });
     },
@@ -614,7 +619,7 @@ export function mlApiServicesProvider(httpService: HttpService) {
     },
 
     mlNodeCount() {
-      return httpService.http<{ count: number }>({
+      return httpService.http<MlNodeCount>({
         path: `${basePath()}/ml_node_count`,
         method: 'GET',
       });

x-pack/plugins/ml/server/lib/node_utils.ts

@@ -0,0 +1,79 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { IScopedClusterClient } from 'kibana/server';
+import { MlNodeCount } from '../../common/types/ml_server_info';
+
+export async function getMlNodeCount(client: IScopedClusterClient): Promise<MlNodeCount> {
+  const { body } = await client.asInternalUser.nodes.info({
+    filter_path: 'nodes.*.attributes',
+  });
+
+  let count = 0;
+  if (typeof body.nodes === 'object') {
+    Object.keys(body.nodes).forEach((k) => {
+      if (body.nodes[k].attributes !== undefined) {
+        const maxOpenJobs = body.nodes[k].attributes['ml.max_open_jobs'];
+        if (maxOpenJobs !== null && maxOpenJobs > 0) {
+          count++;
+        }
+      }
+    });
+  }
+
+  const lazyNodeCount = await getLazyMlNodeCount(client);
+
+  return { count, lazyNodeCount };
+}
+
+export async function getLazyMlNodeCount(client: IScopedClusterClient) {
+  const { body } = await client.asInternalUser.cluster.getSettings({
+    include_defaults: true,
+    filter_path: '**.xpack.ml.max_lazy_ml_nodes',
+  });
+
+  const lazyMlNodesString: string | undefined = (
+    body.defaults ||
+    body.persistent ||
+    body.transient ||
+    {}
+  ).xpack?.ml.max_lazy_ml_nodes;
+
+  const count = lazyMlNodesString === undefined ? 0 : +lazyMlNodesString;
+
+  if (count === 0 || isNaN(count)) {
+    return 0;
+  }
+
+  return count;
+}
+
+export async function countJobsLazyStarting(
+  client: IScopedClusterClient,
+  startingJobsCount: number
+) {
+  const lazyMlNodesCount = await getLazyMlNodeCount(client);
+  const { count: currentMlNodeCount } = await getMlNodeCount(client);
+
+  const availableLazyMlNodes = lazyMlNodesCount ?? lazyMlNodesCount - currentMlNodeCount;
+
+  let lazilyStartingJobsCount = startingJobsCount;
+
+  if (startingJobsCount > availableLazyMlNodes) {
+    if (lazyMlNodesCount > currentMlNodeCount) {
+      lazilyStartingJobsCount = availableLazyMlNodes;
+    }
+  }
+
+  const response = {
+    availableLazyMlNodes,
+    currentMlNodeCount,
+    lazilyStartingJobsCount,
+    totalStartingJobs: startingJobsCount,
+  };
+
+  return response;
+}

x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts

@@ -35,12 +35,12 @@ import {
 import type { MlClient } from '../../lib/ml_client';
 
 export class AnalyticsManager {
-  private _client: IScopedClusterClient['asInternalUser'];
+  private _client: IScopedClusterClient;
   private _mlClient: MlClient;
   private _inferenceModels: TrainedModelConfigResponse[];
   private _jobStats: DataFrameAnalyticsStats[];
 
-  constructor(mlClient: MlClient, client: IScopedClusterClient['asInternalUser']) {
+  constructor(mlClient: MlClient, client: IScopedClusterClient) {
     this._client = client;
     this._mlClient = mlClient;
     this._inferenceModels = [];
@@ -112,7 +112,9 @@ export class AnalyticsManager {
   }
 
   private async getAnalyticsStats() {
-    const resp = await this._mlClient.getDataFrameAnalyticsStats({ size: 1000 });
+    const resp = await this._mlClient.getDataFrameAnalyticsStats<{
+      data_frame_analytics: DataFrameAnalyticsStats[];
+    }>({ size: 1000 });
     const stats = resp?.body?.data_frame_analytics;
     return stats;
   }
@@ -142,15 +144,14 @@ export class AnalyticsManager {
   }
 
   private async getIndexData(index: string) {
-    const indexData = await this._client.indices.get({
+    const indexData = await this._client.asInternalUser.indices.get({
       index,
     });
-
     return indexData?.body;
   }
 
   private async getTransformData(transformId: string) {
-    const transform = await this._client.transform.getTransform({
+    const transform = await this._client.asInternalUser.transform.getTransform({
       transform_id: transformId,
     });
     const transformData = transform?.body?.transforms[0];

x-pack/plugins/ml/server/models/job_service/jobs.ts

@@ -204,6 +204,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
         isNotSingleMetricViewerJobMessage: errorMessage,
         nodeName: job.node ? job.node.name : undefined,
         deleting: job.deleting || undefined,
+        awaitingNodeAssignment: isJobAwaitingNodeAssignment(job),
       };
       if (jobIds.find((j) => j === tempJob.id)) {
         tempJob.fullJob = job;
@@ -519,6 +520,10 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
     return false;
   }
 
+  function isJobAwaitingNodeAssignment(job: CombinedJobWithStats) {
+    return job.node === undefined && job.state === JOB_STATE.OPENING;
+  }
+
   return {
     forceDeleteJob,
     deleteJobs,

x-pack/plugins/ml/server/routes/data_frame_analytics.ts

@@ -44,7 +44,7 @@ function getAnalyticsMap(
   client: IScopedClusterClient,
   idOptions: GetAnalyticsMapArgs
 ) {
-  const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
+  const analytics = new AnalyticsManager(mlClient, client);
   return analytics.getAnalyticsMap(idOptions);
 }
 
@@ -53,7 +53,7 @@ function getExtendedMap(
   client: IScopedClusterClient,
   idOptions: ExtendAnalyticsMapArgs
 ) {
-  const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
+  const analytics = new AnalyticsManager(mlClient, client);
   return analytics.extendAnalyticsMapForAnalyticsJob(idOptions);
 }
 

x-pack/plugins/ml/server/routes/system.ts

@@ -5,12 +5,13 @@
  */
 
 import { schema } from '@kbn/config-schema';
-import { IScopedClusterClient } from 'kibana/server';
+
 import { wrapError } from '../client/error_wrapper';
 import { mlLog } from '../lib/log';
 import { capabilitiesProvider } from '../lib/capabilities';
 import { spacesUtilsProvider } from '../lib/spaces_utils';
 import { RouteInitialization, SystemRouteDeps } from '../types';
+import { getMlNodeCount } from '../lib/node_utils';
 
 /**
  * System routes
@@ -19,25 +20,6 @@ export function systemRoutes(
   { router, mlLicense, routeGuard }: RouteInitialization,
   { getSpaces, cloud, resolveMlCapabilities }: SystemRouteDeps
 ) {
-  async function getNodeCount(client: IScopedClusterClient) {
-    const { body } = await client.asInternalUser.nodes.info({
-      filter_path: 'nodes.*.attributes',
-    });
-
-    let count = 0;
-    if (typeof body.nodes === 'object') {
-      Object.keys(body.nodes).forEach((k) => {
-        if (body.nodes[k].attributes !== undefined) {
-          const maxOpenJobs = body.nodes[k].attributes['ml.max_open_jobs'];
-          if (maxOpenJobs !== null && maxOpenJobs > 0) {
-            count++;
-          }
-        }
-      });
-    }
-    return { count };
-  }
-
   /**
    * @apiGroup SystemRoutes
    *
@@ -156,7 +138,7 @@ export function systemRoutes(
     routeGuard.basicLicenseAPIGuard(async ({ client, response }) => {
       try {
         return response.ok({
-          body: await getNodeCount(client),
+          body: await getMlNodeCount(client),
         });
       } catch (e) {
         return response.customError(wrapError(e));

x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts

@@ -43,6 +43,7 @@ describe('useInstalledSecurityJobs', () => {
       expect(result.current.jobs).toEqual(
         expect.arrayContaining([
           {
+            awaitingNodeAssignment: false,
             datafeedId: 'datafeed-siem-api-rare_process_linux_ecs',
             datafeedIndices: ['auditbeat-*'],
             datafeedState: 'stopped',

x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts

@@ -46,6 +46,7 @@ export const mockOpenedJob: MlSummaryJob = {
   memory_status: 'hard_limit',
   nodeName: 'siem-es',
   processed_record_count: 3425264,
+  awaitingNodeAssignment: false,
 };
 
 export const mockJobsSummaryResponse: MlSummaryJob[] = [
@@ -64,6 +65,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     latestTimestampMs: 1561402325194,
     earliestTimestampMs: 1554327458406,
     isSingleMetricViewerJob: true,
+    awaitingNodeAssignment: false,
   },
   {
     id: 'siem-api-rare_process_linux_ecs',
@@ -79,6 +81,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     latestTimestampMs: 1557434782207,
     earliestTimestampMs: 1557353420495,
     isSingleMetricViewerJob: true,
+    awaitingNodeAssignment: false,
   },
   {
     id: 'siem-api-rare_process_windows_ecs',
@@ -92,6 +95,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     datafeedIndices: ['winlogbeat-*'],
     datafeedState: 'stopped',
     isSingleMetricViewerJob: true,
+    awaitingNodeAssignment: false,
   },
   {
     id: 'siem-api-suspicious_login_activity_ecs',
@@ -105,6 +109,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
     datafeedIndices: ['auditbeat-*'],
     datafeedState: 'stopped',
     isSingleMetricViewerJob: true,
+    awaitingNodeAssignment: false,
   },
 ];
 
@@ -513,6 +518,7 @@ export const mockSecurityJobs: SecurityJob[] = [
     isCompatible: true,
     isInstalled: true,
     isElasticJob: true,
+    awaitingNodeAssignment: false,
   },
   {
     id: 'rare_process_by_host_linux_ecs',
@@ -531,6 +537,7 @@ export const mockSecurityJobs: SecurityJob[] = [
     isCompatible: true,
     isInstalled: true,
     isElasticJob: true,
+    awaitingNodeAssignment: false,
   },
   {
     datafeedId: '',
@@ -549,5 +556,6 @@ export const mockSecurityJobs: SecurityJob[] = [
     isCompatible: false,
     isInstalled: false,
     isElasticJob: true,
+    awaitingNodeAssignment: false,
   },
 ];

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts

@@ -65,6 +65,7 @@ describe('useSecurityJobs', () => {
         memory_status: 'hard_limit',
         moduleId: '',
         processed_record_count: 582251,
+        awaitingNodeAssignment: false,
       };
 
       const { result, waitForNextUpdate } = renderHook(() => useSecurityJobs(false));

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx

@@ -29,6 +29,7 @@ describe('useSecurityJobsHelpers', () => {
         false
       );
       expect(securityJob).toEqual({
+        awaitingNodeAssignment: false,
         datafeedId: '',
         datafeedIndices: [],
         datafeedState: '',

x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx

@@ -42,6 +42,7 @@ export const moduleToSecurityJob = (
     isCompatible,
     isInstalled: false,
     isElasticJob: true,
+    awaitingNodeAssignment: false,
   };
 };
 

x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap

@@ -25,6 +25,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
   items={
     Array [
       Object {
+        "awaitingNodeAssignment": false,
         "datafeedId": "datafeed-linux_anomalous_network_activity_ecs",
         "datafeedIndices": Array [
           "auditbeat-*",
@@ -52,6 +53,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
         "processed_record_count": 32010,
       },
       Object {
+        "awaitingNodeAssignment": false,
         "datafeedId": "datafeed-rare_process_by_host_linux_ecs",
         "datafeedIndices": Array [
           "auditbeat-*",
@@ -76,6 +78,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
         "processed_record_count": 0,
       },
       Object {
+        "awaitingNodeAssignment": false,
         "datafeedId": "",
         "datafeedIndices": Array [],
         "datafeedState": "",

x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap

@@ -28,6 +28,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
         securityJobs={
           Array [
             Object {
+              "awaitingNodeAssignment": false,
               "datafeedId": "datafeed-linux_anomalous_network_activity_ecs",
               "datafeedIndices": Array [
                 "auditbeat-*",
@@ -55,6 +56,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
               "processed_record_count": 32010,
             },
             Object {
+              "awaitingNodeAssignment": false,
               "datafeedId": "datafeed-rare_process_by_host_linux_ecs",
               "datafeedIndices": Array [
                 "auditbeat-*",
@@ -79,6 +81,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
               "processed_record_count": 0,
             },
             Object {
+              "awaitingNodeAssignment": false,
               "datafeedId": "",
               "datafeedIndices": Array [],
               "datafeedState": "",