maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Fixing cloud behaviour when ML nodes are lazily assigned (#90051)

Browse source 
* [ML] Fixing cloud behaviour when ML nodes are lazily assigned

* fixing duplicate id

* updating snapshot

* removing comments

* fixing tests

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
James Gowdy 2021-02-03 17:01:05 +00:00 committed by GitHub
parent d00266b3ff
commit 501f763713
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 386 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts
View file

@ -32,6 +32,7 @@ export interface MlSummaryJob {
deleting?: boolean;
latestTimestampSortValue?: number;
earliestStartTimestampMs?: number;
awaitingNodeAssignment: boolean;
}
export interface AuditMessage {

5
x-pack/plugins/ml/common/types/ml_server_info.ts
View file

@ -31,3 +31,8 @@ export interface MlInfoResponse {
upgrade_mode: boolean;
cloudId?: string;
}
export interface MlNodeCount {
count: number;
lazyNodeCount: number;
}

8
x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
export { JobsAwaitingNodeWarning } from './jobs_awaiting_node_warning';
export { NewJobAwaitingNodeWarning } from './new_job_awaiting_node';

47
x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/jobs_awaiting_node_warning.tsx Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment, FC } from 'react';
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { isCloud } from '../../services/ml_server_info';
interface Props {
jobCount: number;
}
export const JobsAwaitingNodeWarning: FC<Props> = ({ jobCount }) => {
if (isCloud() === false || jobCount === 0) {
return null;
}
return (
<Fragment>
<EuiCallOut
title={
<FormattedMessage
id="xpack.ml.jobsAwaitingNodeWarning.title"
defaultMessage="Awaiting ML node provisioning"
/>
}
color="primary"
iconType="iInCircle"
>
<div>
<FormattedMessage
id="xpack.ml.jobsAwaitingNodeWarning.noMLNodesAvailableDescription"
defaultMessage="There {jobCount, plural, one {is} other {are}} {jobCount, plural, one {# job} other {# jobs}} waiting to be started while ML nodes are being provisioned."
values={{
jobCount,
}}
/>
</div>
</EuiCallOut>
<EuiSpacer size="m" />
</Fragment>
);
};

40
x-pack/plugins/ml/public/application/components/jobs_awaiting_node_warning/new_job_awaiting_node.tsx Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment, FC } from 'react';
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import { JobType } from '../../../../common/types/saved_objects';
interface Props {
jobType: JobType;
}
export const NewJobAwaitingNodeWarning: FC<Props> = () => {
return (
<Fragment>
<EuiCallOut
title={
<FormattedMessage
id="xpack.ml.jobsAwaitingNodeWarning.title"
defaultMessage="Awaiting ML node provisioning"
/>
}
color="primary"
iconType="iInCircle"
>
<div>
<FormattedMessage
id="xpack.ml.newJobAwaitingNodeWarning.noMLNodesAvailableDescription"
defaultMessage="Job cannot be started straight away, an ML node needs to be started. This will happen automatically."
/>
</div>
</EuiCallOut>
<EuiSpacer size="m" />
</Fragment>
);
};

2
x-pack/plugins/ml/public/application/components/node_available_warning/node_available_warning.tsx
View file

@ -41,7 +41,7 @@ export const NodeAvailableWarning: FC = () => {
defaultMessage="You will not be able to create or run jobs."
/>
</div>
{isCloud && id !== null && (
{isCloud() && id !== null && (
<div>
<FormattedMessage
id="xpack.ml.jobsList.nodeAvailableWarning.linkToCloudDescription"

47
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/create_step_footer/create_step_footer.tsx
View file

@ -19,6 +19,7 @@ import { BackToListPanel } from '../back_to_list_panel';
import { ViewResultsPanel } from '../view_results_panel';
import { ProgressStats } from './progress_stats';
import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
import { NewJobAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
export const PROGRESS_REFRESH_INTERVAL_MS = 1000;
@ -41,6 +42,7 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
const [currentProgress, setCurrentProgress] = useState<AnalyticsProgressStats | undefined>(
undefined
);
const [showJobAssignWarning, setShowJobAssignWarning] = useState(false);
const {
services: { notifications },
@ -58,6 +60,10 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
? analyticsStats.data_frame_analytics[0]
: undefined;
setShowJobAssignWarning(
jobStats?.state === DATA_FRAME_TASK_STATE.STARTING && jobStats?.node === undefined
);
if (jobStats !== undefined) {
const progressStats = getDataFrameAnalyticsProgressPhase(jobStats);
@ -106,25 +112,28 @@ export const CreateStepFooter: FC<Props> = ({ jobId, jobType, showProgress }) =>
}, [initialized]);
return (
<EuiFlexGroup direction="column">
<EuiFlexItem grow={false}>
{showProgress && (
<ProgressStats currentProgress={currentProgress} failedJobMessage={failedJobMessage} />
)}
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiHorizontalRule />
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<BackToListPanel />
</EuiFlexItem>
{jobFinished === true && (
<EuiFlexItem grow={false}>
<ViewResultsPanel jobId={jobId} analysisType={jobType} />
</EuiFlexItem>
<>
{showJobAssignWarning && <NewJobAwaitingNodeWarning jobType="data-frame-analytics" />}
<EuiFlexGroup direction="column">
<EuiFlexItem grow={false}>
{showProgress && (
<ProgressStats currentProgress={currentProgress} failedJobMessage={failedJobMessage} />
)}
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiHorizontalRule />
<EuiFlexGroup>
<EuiFlexItem grow={false}>
<BackToListPanel />
</EuiFlexItem>
{jobFinished === true && (
<EuiFlexItem grow={false}>
<ViewResultsPanel jobId={jobId} analysisType={jobType} />
</EuiFlexItem>
)}
</EuiFlexGroup>
</EuiFlexItem>
</EuiFlexGroup>
</>
);
};

4
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/components/analytics_list/analytics_list.tsx
View file

@ -36,6 +36,7 @@ import { AnalyticsEmptyPrompt } from './empty_prompt';
import { useTableSettings } from './use_table_settings';
import { RefreshAnalyticsListButton } from '../refresh_analytics_list_button';
import { ListingPageUrlState } from '../../../../../../../common/types/common';
import { JobsAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
const filters: EuiSearchBarProps['filters'] = [
{
@ -114,6 +115,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
);
const [expandedRowItemIds, setExpandedRowItemIds] = useState<DataFrameAnalyticsId[]>([]);
const [errorMessage, setErrorMessage] = useState<any>(undefined);
const [jobsAwaitingNodeCount, setJobsAwaitingNodeCount] = useState(0);
const disabled =
!checkPermission('canCreateDataFrameAnalytics') ||
@ -124,6 +126,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
setAnalyticsStats,
setErrorMessage,
setIsInitialized,
setJobsAwaitingNodeCount,
blockRefresh,
isManagementTable
);
@ -261,6 +264,7 @@ export const DataFrameAnalyticsList: FC<Props> = ({
<div data-test-subj="mlAnalyticsJobList">
{modals}
{!isManagementTable && <EuiSpacer size="m" />}
<JobsAwaitingNodeWarning jobCount={jobsAwaitingNodeCount} />
<EuiFlexGroup justifyContent="spaceBetween">
{!isManagementTable && stats}
{isManagementTable && managementStats}

9
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_management/services/analytics_service/get_analytics.ts
View file

@ -26,6 +26,7 @@ import {
} from '../../components/analytics_list/common';
import { AnalyticStatsBarStats } from '../../../../../components/stats_bar';
import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
import { DATA_FRAME_TASK_STATE } from '../../../../../../../common/constants/data_frame_analytics';
export const isGetDataFrameAnalyticsStatsResponseOk = (
arg: any
@ -106,6 +107,7 @@ export const getAnalyticsFactory = (
React.SetStateAction<GetDataFrameAnalyticsStatsResponseError | undefined>
>,
setIsInitialized: React.Dispatch<React.SetStateAction<boolean>>,
setJobsAwaitingNodeCount: React.Dispatch<React.SetStateAction<number>>,
blockRefresh: boolean,
isManagementTable: boolean
): GetAnalytics => {
@ -134,6 +136,8 @@ export const getAnalyticsFactory = (
? getAnalyticsJobsStats(analyticsStats)
: undefined;
let jobsAwaitingNodeCount = 0;
const tableRows = analyticsConfigs.data_frame_analytics.reduce(
(reducedtableRows, config) => {
const stats = isGetDataFrameAnalyticsStatsResponseOk(analyticsStats)
@ -146,6 +150,10 @@ export const getAnalyticsFactory = (
return reducedtableRows;
}
if (stats.state === DATA_FRAME_TASK_STATE.STARTING && stats.node === undefined) {
jobsAwaitingNodeCount++;
}
// Table with expandable rows requires `id` on the outer most level
reducedtableRows.push({
checkpointing: {},
@ -166,6 +174,7 @@ export const getAnalyticsFactory = (
setAnalyticsStats(analyticsStatsResult);
setErrorMessage(undefined);
setIsInitialized(true);
setJobsAwaitingNodeCount(jobsAwaitingNodeCount);
refreshAnalyticsList$.next(REFRESH_ANALYTICS_LIST_STATE.IDLE);
} catch (e) {
// An error is followed immediately by setting the state to idle.

18
x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js
View file

@ -32,6 +32,7 @@ import { MultiJobActions } from '../multi_job_actions';
import { NewJobButton } from '../new_job_button';
import { JobStatsBar } from '../jobs_stats_bar';
import { NodeAvailableWarning } from '../../../../components/node_available_warning';
import { JobsAwaitingNodeWarning } from '../../../../components/jobs_awaiting_node_warning';
import { SavedObjectsWarning } from '../../../../components/saved_objects_warning';
import { DatePickerWrapper } from '../../../../components/navigation_menu/date_picker_wrapper';
import { UpgradeWarning } from '../../../../components/upgrade';
@ -56,6 +57,7 @@ export class JobsListView extends Component {
itemIdToExpandedRowMap: {},
filterClauses: [],
deletingJobIds: [],
jobsAwaitingNodeCount: 0,
};
this.spacesEnabled = props.spacesEnabled ?? false;
@ -272,6 +274,7 @@ export class JobsListView extends Component {
spaces = allSpaces['anomaly-detector'];
}
let jobsAwaitingNodeCount = 0;
const jobs = await ml.jobs.jobsSummary(expandedJobsIds);
const fullJobsList = {};
const jobsSummaryList = jobs.map((job) => {
@ -287,11 +290,21 @@ export class JobsListView extends Component {
spaces[job.id] !== undefined
? spaces[job.id]
: [];
if (job.awaitingNodeAssignment === true) {
jobsAwaitingNodeCount++;
}
return job;
});
const filteredJobsSummaryList = filterJobs(jobsSummaryList, this.state.filterClauses);
this.setState(
{ jobsSummaryList, filteredJobsSummaryList, fullJobsList, loading: false },
{
jobsSummaryList,
filteredJobsSummaryList,
fullJobsList,
loading: false,
jobsAwaitingNodeCount,
},
() => {
this.refreshSelectedJobs();
}
@ -407,7 +420,7 @@ export class JobsListView extends Component {
}
renderJobsListComponents() {
const { isRefreshing, loading, jobsSummaryList } = this.state;
const { isRefreshing, loading, jobsSummaryList, jobsAwaitingNodeCount } = this.state;
const jobIds = jobsSummaryList.map((j) => j.id);
return (
@ -440,6 +453,7 @@ export class JobsListView extends Component {
</EuiPageHeader>
<NodeAvailableWarning />
<JobsAwaitingNodeWarning jobCount={jobsAwaitingNodeCount} />
<SavedObjectsWarning jobType="anomaly-detector" />
<UpgradeWarning />

51
x-pack/plugins/ml/public/application/jobs/new_job/common/job_runner/job_runner.ts
View file

@ -11,12 +11,14 @@ import { JobCreator } from '../job_creator';
import { DatafeedId, JobId } from '../../../../../../common/types/anomaly_detection_jobs';
import { DATAFEED_STATE } from '../../../../../../common/constants/states';
const REFRESH_INTERVAL_MS = 100;
const REFRESH_INTERVAL_MS = 250;
const NODE_ASSIGNMENT_CHECK_REFRESH_INTERVAL_MS = 2000;
const TARGET_PROGRESS_DELTA = 2;
const REFRESH_RATE_ADJUSTMENT_DELAY_MS = 2000;
type Progress = number;
export type ProgressSubscriber = (progress: number) => void;
export type JobAssignmentSubscriber = (assigned: boolean) => void;
export class JobRunner {
private _jobId: JobId;
@ -35,6 +37,8 @@ export class JobRunner {
private _datafeedStartTime: number = 0;
private _performRefreshRateAdjustment: boolean = false;
private _jobAssignedToNode: boolean = false;
private _jobAssignedToNode$: BehaviorSubject<boolean>;
constructor(jobCreator: JobCreator) {
this._jobId = jobCreator.jobId;
@ -45,6 +49,7 @@ export class JobRunner {
this._stopRefreshPoll = jobCreator.stopAllRefreshPolls;
this._progress$ = new BehaviorSubject(this._percentageComplete);
this._jobAssignedToNode$ = new BehaviorSubject(this._jobAssignedToNode);
this._subscribers = jobCreator.subscribers;
}
@ -62,7 +67,9 @@ export class JobRunner {
private async openJob(): Promise<void> {
try {
await mlJobService.openJob(this._jobId);
const { node }: { node?: string } = await mlJobService.openJob(this._jobId);
this._jobAssignedToNode = node !== undefined && node.length > 0;
this._jobAssignedToNode$.next(this._jobAssignedToNode);
} catch (error) {
throw error;
}
@ -94,7 +101,7 @@ export class JobRunner {
this._datafeedState = DATAFEED_STATE.STARTED;
this._percentageComplete = 0;
const check = async () => {
const checkProgress = async () => {
const { isRunning, progress: prog, isJobClosed } = await this.getProgress();
// if the progress has reached 100% but the job is still running,
@ -109,7 +116,9 @@ export class JobRunner {
if ((isRunning === true || isJobClosed === false) && this._stopRefreshPoll.stop === false) {
setTimeout(async () => {
await check();
if (this._stopRefreshPoll.stop === false) {
await checkProgress();
}
}, this._refreshInterval);
} else {
// job has finished running, set progress to 100%
@ -121,11 +130,30 @@ export class JobRunner {
subscriptions.forEach((s) => s.unsubscribe());
}
};
const checkJobIsAssigned = async () => {
this._jobAssignedToNode = await this._isJobAssigned();
this._jobAssignedToNode$.next(this._jobAssignedToNode);
if (this._jobAssignedToNode === true) {
await checkProgress();
} else {
setTimeout(async () => {
if (this._stopRefreshPoll.stop === false) {
await checkJobIsAssigned();
}
}, NODE_ASSIGNMENT_CHECK_REFRESH_INTERVAL_MS);
}
};
// wait for the first check to run and then return success.
// all subsequent checks will update the observable
if (pollProgress === true) {
await check();
if (this._jobAssignedToNode === true) {
await checkProgress();
} else {
await checkJobIsAssigned();
}
}
return started;
} catch (error) {
throw error;
@ -159,6 +187,11 @@ export class JobRunner {
}
}
private async _isJobAssigned(): Promise<boolean> {
const { jobs } = await ml.getJobStats({ jobId: this._jobId });
return jobs.length > 0 && jobs[0].node !== undefined;
}
public async startDatafeed() {
return await this._startDatafeed(this._start, this._end, true);
}
@ -185,4 +218,12 @@ export class JobRunner {
const { isRunning } = await this.getProgress();
return isRunning;
}
public isJobAssignedToNode() {
return this._jobAssignedToNode;
}
public subscribeToJobAssignment(func: JobAssignmentSubscriber) {
return this._jobAssignedToNode$.subscribe(func);
}
}

18
x-pack/plugins/ml/public/application/jobs/new_job/pages/components/summary_step/summary.tsx
View file

@ -5,6 +5,7 @@
*/
import React, { Fragment, FC, useContext, useState, useEffect } from 'react';
import { Subscription } from 'rxjs';
import {
EuiButton,
EuiButtonEmpty,
@ -29,6 +30,7 @@ import { DetectorChart } from './components/detector_chart';
import { JobProgress } from './components/job_progress';
import { PostSaveOptions } from './components/post_save_options';
import { StartDatafeedSwitch } from './components/start_datafeed_switch';
import { NewJobAwaitingNodeWarning } from '../../../../../components/jobs_awaiting_node_warning';
import { toastNotificationServiceProvider } from '../../../../../services/toast_notification_service';
import {
convertToAdvancedJob,
@ -55,6 +57,7 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
const [isValid, setIsValid] = useState(jobValidator.validationSummary.basic);
const [jobRunner, setJobRunner] = useState<JobRunner | null>(null);
const [startDatafeed, setStartDatafeed] = useState(true);
const [showJobAssignWarning, setShowJobAssignWarning] = useState(false);
const isAdvanced = isAdvancedJobCreator(jobCreator);
const jsonEditorMode = isAdvanced ? EDITOR_MODE.EDITABLE : EDITOR_MODE.READONLY;
@ -63,6 +66,20 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
jobCreator.subscribeToProgress(setProgress);
}, []);
useEffect(() => {
let s: Subscription | null = null;
if (jobRunner !== null) {
s = jobRunner.subscribeToJobAssignment((assigned: boolean) =>
setShowJobAssignWarning(!assigned)
);
}
return () => {
if (s !== null) {
s?.unsubscribe();
}
};
}, [jobRunner]);
async function start() {
setCreatingJob(true);
if (isAdvanced) {
@ -155,6 +172,7 @@ export const SummaryStep: FC<StepProps> = ({ setCurrentStep, isCurrentStep }) =>
)}
<EuiHorizontalRule />
{showJobAssignWarning && <NewJobAwaitingNodeWarning jobType="anomaly-detector" />}
<EuiFlexGroup>
{progress < 100 && (
<Fragment>

15
x-pack/plugins/ml/public/application/ml_nodes_check/check_ml_nodes.ts
View file

@ -5,14 +5,16 @@
*/
import { ml } from '../services/ml_api_service';
import { MlNodeCount } from '../../../common/types/ml_server_info';
let mlNodeCount: number = 0;
let lazyMlNodeCount: number = 0;
let userHasPermissionToViewMlNodeCount: boolean = false;
export async function checkMlNodesAvailable(redirectToJobsManagementPage: () => Promise<void>) {
try {
const nodes = await getMlNodeCount();
if (nodes.count !== undefined && nodes.count > 0) {
const { count, lazyNodeCount } = await getMlNodeCount();
if (count > 0 || lazyNodeCount > 0) {
Promise.resolve();
} else {
throw Error('Cannot load count of ML nodes');
@ -25,23 +27,24 @@ export async function checkMlNodesAvailable(redirectToJobsManagementPage: () =>
}
}
export async function getMlNodeCount() {
export async function getMlNodeCount(): Promise<MlNodeCount> {
try {
const nodes = await ml.mlNodeCount();
mlNodeCount = nodes.count;
lazyMlNodeCount = nodes.lazyNodeCount;
userHasPermissionToViewMlNodeCount = true;
return Promise.resolve(nodes);
return nodes;
} catch (error) {
mlNodeCount = 0;
if (error.statusCode === 403) {
userHasPermissionToViewMlNodeCount = false;
}
return Promise.resolve({ count: 0 });
return { count: 0, lazyNodeCount: 0 };
}
}
export function mlNodesAvailable() {
return mlNodeCount !== 0;
return mlNodeCount !== 0 || lazyMlNodeCount !== 0;
}
export function permissionToViewMlNodeCount() {

4
x-pack/plugins/ml/public/application/overview/components/analytics_panel/analytics_panel.tsx
View file

@ -28,8 +28,9 @@ import { ML_PAGES } from '../../../../../common/constants/ml_url_generator';
interface Props {
jobCreationDisabled: boolean;
setLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
}
export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled }) => {
export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled, setLazyJobCount }) => {
const [analytics, setAnalytics] = useState<DataFrameAnalyticsListRow[]>([]);
const [analyticsStats, setAnalyticsStats] = useState<AnalyticStatsBarStats | undefined>(
undefined
@ -52,6 +53,7 @@ export const AnalyticsPanel: FC<Props> = ({ jobCreationDisabled }) => {
setAnalyticsStats,
setErrorMessage,
setIsInitialized,
setLazyJobCount,
false,
false
);

8
x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/anomaly_detection_panel.tsx
View file

@ -51,9 +51,10 @@ function getDefaultAnomalyScores(groups: Group[]): MaxScoresByGroup {
interface Props {
jobCreationDisabled: boolean;
setLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
}
export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled, setLazyJobCount }) => {
const {
services: { notifications },
} = useMlKibana();
@ -84,10 +85,14 @@ export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
const loadJobs = async () => {
setIsLoading(true);
let lazyJobCount = 0;
try {
const jobsResult: MlSummaryJobs = await ml.jobs.jobsSummary([]);
const jobsSummaryList = jobsResult.map((job: MlSummaryJob) => {
job.latestTimestampSortValue = job.latestTimestampMs || 0;
if (job.awaitingNodeAssignment) {
lazyJobCount++;
}
return job;
});
const { groups: jobsGroups, count } = getGroupsFromJobs(jobsSummaryList);
@ -100,6 +105,7 @@ export const AnomalyDetectionPanel: FC<Props> = ({ jobCreationDisabled }) => {
setGroups(jobsGroups);
setJobsList(jobsWithTimerange);
loadMaxAnomalyScores(jobsGroups);
setLazyJobCount(lazyJobCount);
} catch (e) {
setErrorMessage(e.message !== undefined ? e.message : JSON.stringify(e));
setIsLoading(false);

14
x-pack/plugins/ml/public/application/overview/components/content.tsx
View file

@ -12,20 +12,30 @@ import { AnalyticsPanel } from './analytics_panel';
interface Props {
createAnomalyDetectionJobDisabled: boolean;
createAnalyticsJobDisabled: boolean;
setAdLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
setDfaLazyJobCount: React.Dispatch<React.SetStateAction<number>>;
}
// Fetch jobs and determine what to show
export const OverviewContent: FC<Props> = ({
createAnomalyDetectionJobDisabled,
createAnalyticsJobDisabled,
setAdLazyJobCount,
setDfaLazyJobCount,
}) => (
<EuiFlexItem grow={3}>
<EuiFlexGroup direction="column">
<EuiFlexItem grow={false}>
<AnomalyDetectionPanel jobCreationDisabled={createAnomalyDetectionJobDisabled} />
<AnomalyDetectionPanel
jobCreationDisabled={createAnomalyDetectionJobDisabled}
setLazyJobCount={setAdLazyJobCount}
/>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<AnalyticsPanel jobCreationDisabled={createAnalyticsJobDisabled} />
<AnalyticsPanel
jobCreationDisabled={createAnalyticsJobDisabled}
setLazyJobCount={setDfaLazyJobCount}
/>
</EuiFlexItem>
</EuiFlexGroup>
</EuiFlexItem>

10
x-pack/plugins/ml/public/application/overview/overview_page.tsx
View file

@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import React, { Fragment, FC } from 'react';
import React, { Fragment, FC, useState } from 'react';
import { EuiFlexGroup, EuiPage, EuiPageBody } from '@elastic/eui';
import { checkPermission } from '../capabilities/check_capabilities';
import { mlNodesAvailable } from '../ml_nodes_check/check_ml_nodes';
@ -12,6 +12,7 @@ import { NavigationMenu } from '../components/navigation_menu';
import { OverviewSideBar } from './components/sidebar';
import { OverviewContent } from './components/content';
import { NodeAvailableWarning } from '../components/node_available_warning';
import { JobsAwaitingNodeWarning } from '../components/jobs_awaiting_node_warning';
import { SavedObjectsWarning } from '../components/saved_objects_warning';
import { UpgradeWarning } from '../components/upgrade';
import { HelpMenu } from '../components/help_menu';
@ -27,12 +28,17 @@ export const OverviewPage: FC = () => {
services: { docLinks },
} = useMlKibana();
const helpLink = docLinks.links.ml.guide;
const [adLazyJobCount, setAdLazyJobCount] = useState(0);
const [dfaLazyJobCount, setDfaLazyJobCount] = useState(0);
return (
<Fragment>
<NavigationMenu tabId="overview" />
<EuiPage data-test-subj="mlPageOverview">
<EuiPageBody>
<NodeAvailableWarning />
<JobsAwaitingNodeWarning jobCount={adLazyJobCount + dfaLazyJobCount} />
<SavedObjectsWarning />
<UpgradeWarning />
@ -41,6 +47,8 @@ export const OverviewPage: FC = () => {
<OverviewContent
createAnomalyDetectionJobDisabled={disableCreateAnomalyDetectionJob}
createAnalyticsJobDisabled={disableCreateAnalyticsButton}
setAdLazyJobCount={setAdLazyJobCount}
setDfaLazyJobCount={setDfaLazyJobCount}
/>
</EuiFlexGroup>
</EuiPageBody>

13
x-pack/plugins/ml/public/application/services/ml_api_service/index.ts
View file

@ -15,12 +15,17 @@ import { resultsApiProvider } from './results';
import { jobsApiProvider } from './jobs';
import { fileDatavisualizer } from './datavisualizer';
import { savedObjectsApiProvider } from './saved_objects';
import { MlServerDefaults, MlServerLimits } from '../../../../common/types/ml_server_info';
import {
MlServerDefaults,
MlServerLimits,
MlNodeCount,
} from '../../../../common/types/ml_server_info';
import { MlCapabilitiesResponse } from '../../../../common/types/capabilities';
import { Calendar, CalendarId, UpdateCalendar } from '../../../../common/types/calendars';
import {
Job,
JobStats,
Datafeed,
CombinedJob,
Detector,
@ -116,14 +121,14 @@ export function mlApiServicesProvider(httpService: HttpService) {
return {
getJobs(obj?: { jobId?: string }) {
const jobId = obj && obj.jobId ? `/${obj.jobId}` : '';
return httpService.http<any>({
return httpService.http<{ jobs: Job[]; count: number }>({
path: `${basePath()}/anomaly_detectors${jobId}`,
});
},
getJobStats(obj: { jobId?: string }) {
const jobId = obj && obj.jobId ? `/${obj.jobId}` : '';
return httpService.http<any>({
return httpService.http<{ jobs: JobStats[]; count: number }>({
path: `${basePath()}/anomaly_detectors${jobId}/_stats`,
});
},
@ -614,7 +619,7 @@ export function mlApiServicesProvider(httpService: HttpService) {
},
mlNodeCount() {
return httpService.http<{ count: number }>({
return httpService.http<MlNodeCount>({
path: `${basePath()}/ml_node_count`,
method: 'GET',
});

79
x-pack/plugins/ml/server/lib/node_utils.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { IScopedClusterClient } from 'kibana/server';
import { MlNodeCount } from '../../common/types/ml_server_info';
export async function getMlNodeCount(client: IScopedClusterClient): Promise<MlNodeCount> {
const { body } = await client.asInternalUser.nodes.info({
filter_path: 'nodes.*.attributes',
});
let count = 0;
if (typeof body.nodes === 'object') {
Object.keys(body.nodes).forEach((k) => {
if (body.nodes[k].attributes !== undefined) {
const maxOpenJobs = body.nodes[k].attributes['ml.max_open_jobs'];
if (maxOpenJobs !== null && maxOpenJobs > 0) {
count++;
}
}
});
}
const lazyNodeCount = await getLazyMlNodeCount(client);
return { count, lazyNodeCount };
}
export async function getLazyMlNodeCount(client: IScopedClusterClient) {
const { body } = await client.asInternalUser.cluster.getSettings({
include_defaults: true,
filter_path: '**.xpack.ml.max_lazy_ml_nodes',
});
const lazyMlNodesString: string | undefined = (
body.defaults ||
body.persistent ||
body.transient ||
{}
).xpack?.ml.max_lazy_ml_nodes;
const count = lazyMlNodesString === undefined ? 0 : +lazyMlNodesString;
if (count === 0 || isNaN(count)) {
return 0;
}
return count;
}
export async function countJobsLazyStarting(
client: IScopedClusterClient,
startingJobsCount: number
) {
const lazyMlNodesCount = await getLazyMlNodeCount(client);
const { count: currentMlNodeCount } = await getMlNodeCount(client);
const availableLazyMlNodes = lazyMlNodesCount ?? lazyMlNodesCount - currentMlNodeCount;
let lazilyStartingJobsCount = startingJobsCount;
if (startingJobsCount > availableLazyMlNodes) {
if (lazyMlNodesCount > currentMlNodeCount) {
lazilyStartingJobsCount = availableLazyMlNodes;
}
}
const response = {
availableLazyMlNodes,
currentMlNodeCount,
lazilyStartingJobsCount,
totalStartingJobs: startingJobsCount,
};
return response;
}

13
x-pack/plugins/ml/server/models/data_frame_analytics/analytics_manager.ts
View file

@ -35,12 +35,12 @@ import {
import type { MlClient } from '../../lib/ml_client';
export class AnalyticsManager {
private _client: IScopedClusterClient['asInternalUser'];
private _client: IScopedClusterClient;
private _mlClient: MlClient;
private _inferenceModels: TrainedModelConfigResponse[];
private _jobStats: DataFrameAnalyticsStats[];
constructor(mlClient: MlClient, client: IScopedClusterClient['asInternalUser']) {
constructor(mlClient: MlClient, client: IScopedClusterClient) {
this._client = client;
this._mlClient = mlClient;
this._inferenceModels = [];
@ -112,7 +112,9 @@ export class AnalyticsManager {
}
private async getAnalyticsStats() {
const resp = await this._mlClient.getDataFrameAnalyticsStats({ size: 1000 });
const resp = await this._mlClient.getDataFrameAnalyticsStats<{
data_frame_analytics: DataFrameAnalyticsStats[];
}>({ size: 1000 });
const stats = resp?.body?.data_frame_analytics;
return stats;
}
@ -142,15 +144,14 @@ export class AnalyticsManager {
}
private async getIndexData(index: string) {
const indexData = await this._client.indices.get({
const indexData = await this._client.asInternalUser.indices.get({
index,
});
return indexData?.body;
}
private async getTransformData(transformId: string) {
const transform = await this._client.transform.getTransform({
const transform = await this._client.asInternalUser.transform.getTransform({
transform_id: transformId,
});
const transformData = transform?.body?.transforms[0];

5
x-pack/plugins/ml/server/models/job_service/jobs.ts
View file

@ -204,6 +204,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
isNotSingleMetricViewerJobMessage: errorMessage,
nodeName: job.node ? job.node.name : undefined,
deleting: job.deleting || undefined,
awaitingNodeAssignment: isJobAwaitingNodeAssignment(job),
};
if (jobIds.find((j) => j === tempJob.id)) {
tempJob.fullJob = job;
@ -519,6 +520,10 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
return false;
}
function isJobAwaitingNodeAssignment(job: CombinedJobWithStats) {
return job.node === undefined && job.state === JOB_STATE.OPENING;
}
return {
forceDeleteJob,
deleteJobs,

4
x-pack/plugins/ml/server/routes/data_frame_analytics.ts
View file

@ -44,7 +44,7 @@ function getAnalyticsMap(
client: IScopedClusterClient,
idOptions: GetAnalyticsMapArgs
) {
const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
const analytics = new AnalyticsManager(mlClient, client);
return analytics.getAnalyticsMap(idOptions);
}
@ -53,7 +53,7 @@ function getExtendedMap(
client: IScopedClusterClient,
idOptions: ExtendAnalyticsMapArgs
) {
const analytics = new AnalyticsManager(mlClient, client.asInternalUser);
const analytics = new AnalyticsManager(mlClient, client);
return analytics.extendAnalyticsMapForAnalyticsJob(idOptions);
}

24
x-pack/plugins/ml/server/routes/system.ts
View file

@ -5,12 +5,13 @@
*/
import { schema } from '@kbn/config-schema';
import { IScopedClusterClient } from 'kibana/server';
import { wrapError } from '../client/error_wrapper';
import { mlLog } from '../lib/log';
import { capabilitiesProvider } from '../lib/capabilities';
import { spacesUtilsProvider } from '../lib/spaces_utils';
import { RouteInitialization, SystemRouteDeps } from '../types';
import { getMlNodeCount } from '../lib/node_utils';
/**
* System routes
@ -19,25 +20,6 @@ export function systemRoutes(
{ router, mlLicense, routeGuard }: RouteInitialization,
{ getSpaces, cloud, resolveMlCapabilities }: SystemRouteDeps
) {
async function getNodeCount(client: IScopedClusterClient) {
const { body } = await client.asInternalUser.nodes.info({
filter_path: 'nodes.*.attributes',
});
let count = 0;
if (typeof body.nodes === 'object') {
Object.keys(body.nodes).forEach((k) => {
if (body.nodes[k].attributes !== undefined) {
const maxOpenJobs = body.nodes[k].attributes['ml.max_open_jobs'];
if (maxOpenJobs !== null && maxOpenJobs > 0) {
count++;
}
}
});
}
return { count };
}
/**
* @apiGroup SystemRoutes
*
@ -156,7 +138,7 @@ export function systemRoutes(
routeGuard.basicLicenseAPIGuard(async ({ client, response }) => {
try {
return response.ok({
body: await getNodeCount(client),
body: await getMlNodeCount(client),
});
} catch (e) {
return response.customError(wrapError(e));

1
x-pack/plugins/security_solution/public/common/components/ml/hooks/use_installed_security_jobs.test.ts
View file

@ -43,6 +43,7 @@ describe('useInstalledSecurityJobs', () => {
expect(result.current.jobs).toEqual(
expect.arrayContaining([
{
awaitingNodeAssignment: false,
datafeedId: 'datafeed-siem-api-rare_process_linux_ecs',
datafeedIndices: ['auditbeat-*'],
datafeedState: 'stopped',

8
x-pack/plugins/security_solution/public/common/components/ml_popover/api.mock.ts
View file

@ -46,6 +46,7 @@ export const mockOpenedJob: MlSummaryJob = {
memory_status: 'hard_limit',
nodeName: 'siem-es',
processed_record_count: 3425264,
awaitingNodeAssignment: false,
};
export const mockJobsSummaryResponse: MlSummaryJob[] = [
@ -64,6 +65,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
latestTimestampMs: 1561402325194,
earliestTimestampMs: 1554327458406,
isSingleMetricViewerJob: true,
awaitingNodeAssignment: false,
},
{
id: 'siem-api-rare_process_linux_ecs',
@ -79,6 +81,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
latestTimestampMs: 1557434782207,
earliestTimestampMs: 1557353420495,
isSingleMetricViewerJob: true,
awaitingNodeAssignment: false,
},
{
id: 'siem-api-rare_process_windows_ecs',
@ -92,6 +95,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
datafeedIndices: ['winlogbeat-*'],
datafeedState: 'stopped',
isSingleMetricViewerJob: true,
awaitingNodeAssignment: false,
},
{
id: 'siem-api-suspicious_login_activity_ecs',
@ -105,6 +109,7 @@ export const mockJobsSummaryResponse: MlSummaryJob[] = [
datafeedIndices: ['auditbeat-*'],
datafeedState: 'stopped',
isSingleMetricViewerJob: true,
awaitingNodeAssignment: false,
},
];
@ -513,6 +518,7 @@ export const mockSecurityJobs: SecurityJob[] = [
isCompatible: true,
isInstalled: true,
isElasticJob: true,
awaitingNodeAssignment: false,
},
{
id: 'rare_process_by_host_linux_ecs',
@ -531,6 +537,7 @@ export const mockSecurityJobs: SecurityJob[] = [
isCompatible: true,
isInstalled: true,
isElasticJob: true,
awaitingNodeAssignment: false,
},
{
datafeedId: '',
@ -549,5 +556,6 @@ export const mockSecurityJobs: SecurityJob[] = [
isCompatible: false,
isInstalled: false,
isElasticJob: true,
awaitingNodeAssignment: false,
},
];

1
x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs.test.ts
View file

@ -65,6 +65,7 @@ describe('useSecurityJobs', () => {
memory_status: 'hard_limit',
moduleId: '',
processed_record_count: 582251,
awaitingNodeAssignment: false,
};
const { result, waitForNextUpdate } = renderHook(() => useSecurityJobs(false));

1
x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.test.tsx
View file

@ -29,6 +29,7 @@ describe('useSecurityJobsHelpers', () => {
false
);
expect(securityJob).toEqual({
awaitingNodeAssignment: false,
datafeedId: '',
datafeedIndices: [],
datafeedState: '',

1
x-pack/plugins/security_solution/public/common/components/ml_popover/hooks/use_security_jobs_helpers.tsx
View file

@ -42,6 +42,7 @@ export const moduleToSecurityJob = (
isCompatible,
isInstalled: false,
isElasticJob: true,
awaitingNodeAssignment: false,
};
};

3
x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/__snapshots__/jobs_table.test.tsx.snap generated
View file

@ -25,6 +25,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
items={
Array [
Object {
"awaitingNodeAssignment": false,
"datafeedId": "datafeed-linux_anomalous_network_activity_ecs",
"datafeedIndices": Array [
"auditbeat-*",
@ -52,6 +53,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
"processed_record_count": 32010,
},
Object {
"awaitingNodeAssignment": false,
"datafeedId": "datafeed-rare_process_by_host_linux_ecs",
"datafeedIndices": Array [
"auditbeat-*",
@ -76,6 +78,7 @@ exports[`JobsTableComponent renders correctly against snapshot 1`] = `
"processed_record_count": 0,
},
Object {
"awaitingNodeAssignment": false,
"datafeedId": "",
"datafeedIndices": Array [],
"datafeedState": "",

3
x-pack/plugins/security_solution/public/common/components/ml_popover/jobs_table/filters/__snapshots__/jobs_table_filters.test.tsx.snap generated
View file

@ -28,6 +28,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
securityJobs={
Array [
Object {
"awaitingNodeAssignment": false,
"datafeedId": "datafeed-linux_anomalous_network_activity_ecs",
"datafeedIndices": Array [
"auditbeat-*",
@ -55,6 +56,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
"processed_record_count": 32010,
},
Object {
"awaitingNodeAssignment": false,
"datafeedId": "datafeed-rare_process_by_host_linux_ecs",
"datafeedIndices": Array [
"auditbeat-*",
@ -79,6 +81,7 @@ exports[`JobsTableFilters renders correctly against snapshot 1`] = `
"processed_record_count": 0,
},
Object {
"awaitingNodeAssignment": false,
"datafeedId": "",
"datafeedIndices": Array [],
"datafeedState": "",