maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Fix bytes formatting and default message in the Anomaly detection jobs health rule type (#110069) (#110111)

Browse source 
Co-authored-by: Dima Arnautov <dmitrii.arnautov@elastic.co>
This commit is contained in:
Kibana Machine 2021-08-25 15:27:57 -04:00 committed by GitHub
parent 0360fc13d0
commit 132a67cd8f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 54 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
x-pack/plugins/ml/public/alerting/jobs_health_rule/register_jobs_health_alerting_rule.ts
View file

@ -92,14 +92,18 @@ export function registerJobsHealthAlertingRule(
\\{\\{#context.results\\}\\}
Job ID: \\{\\{job_id\\}\\}
\\{\\{#datafeed_id\\}\\}Datafeed ID: \\{\\{datafeed_id\\}\\}
\\{\\{/datafeed_id\\}\\} \\{\\{#datafeed_state\\}\\}Datafeed state: \\{\\{datafeed_state\\}\\}
\\{\\{/datafeed_state\\}\\} \\{\\{#memory_status\\}\\}Memory status: \\{\\{memory_status\\}\\}
\\{\\{/memory_status\\}\\} \\{\\{#log_time\\}\\}Memory logging time: \\{\\{log_time\\}\\}
\\{\\{/log_time\\}\\} \\{\\{#failed_category_count\\}\\}Failed category count: \\{\\{failed_category_count\\}\\}
\\{\\{/failed_category_count\\}\\} \\{\\{#annotation\\}\\}Annotation: \\{\\{annotation\\}\\}
\\{\\{/annotation\\}\\} \\{\\{#missed_docs_count\\}\\}Number of missed documents: \\{\\{missed_docs_count\\}\\}
\\{\\{/missed_docs_count\\}\\} \\{\\{#end_timestamp\\}\\}Latest finalized bucket with missing docs: \\{\\{end_timestamp\\}\\}
\\{\\{/end_timestamp\\}\\} \\{\\{#errors\\}\\}Error message: \\{\\{message\\}\\} \\{\\{/errors\\}\\}
\\{\\{/datafeed_id\\}\\}\\{\\{#datafeed_state\\}\\}Datafeed state: \\{\\{datafeed_state\\}\\}
\\{\\{/datafeed_state\\}\\}\\{\\{#memory_status\\}\\}Memory status: \\{\\{memory_status\\}\\}
\\{\\{/memory_status\\}\\}\\{\\{#model_bytes\\}\\}Model size: \\{\\{model_bytes\\}\\}
\\{\\{/model_bytes\\}\\}\\{\\{#model_bytes_memory_limit\\}\\}Model memory limit: \\{\\{model_bytes_memory_limit\\}\\}
\\{\\{/model_bytes_memory_limit\\}\\}\\{\\{#peak_model_bytes\\}\\}Peak model bytes: \\{\\{peak_model_bytes\\}\\}
\\{\\{/peak_model_bytes\\}\\}\\{\\{#model_bytes_exceeded\\}\\}Model exceeded: \\{\\{model_bytes_exceeded\\}\\}
\\{\\{/model_bytes_exceeded\\}\\}\\{\\{#log_time\\}\\}Memory logging time: \\{\\{log_time\\}\\}
\\{\\{/log_time\\}\\}\\{\\{#failed_category_count\\}\\}Failed category count: \\{\\{failed_category_count\\}\\}
\\{\\{/failed_category_count\\}\\}\\{\\{#annotation\\}\\}Annotation: \\{\\{annotation\\}\\}
\\{\\{/annotation\\}\\}\\{\\{#missed_docs_count\\}\\}Number of missed documents: \\{\\{missed_docs_count\\}\\}
\\{\\{/missed_docs_count\\}\\}\\{\\{#end_timestamp\\}\\}Latest finalized bucket with missing docs: \\{\\{end_timestamp\\}\\}
\\{\\{/end_timestamp\\}\\}\\{\\{#errors\\}\\}Error message: \\{\\{message\\}\\} \\{\\{/errors\\}\\}
\\{\\{/context.results\\}\\}
`,
}

29
x-pack/plugins/ml/server/lib/alerts/jobs_health_service.test.ts
View file

@ -93,6 +93,10 @@ describe('JobsHealthService', () => {
model_size_stats: {
memory_status: j === 'test_job_01' ? 'hard_limit' : 'ok',
log_time: 1626935914540,
model_bytes: 1000000,
model_bytes_memory_limit: 800000,
peak_model_bytes: 1000000,
model_bytes_exceeded: 200000,
},
};
}) as MlJobStats,
@ -162,12 +166,21 @@ describe('JobsHealthService', () => {
const getFieldsFormatRegistry = jest.fn().mockImplementation(() => {
return Promise.resolve({
deserialize: jest.fn().mockImplementation(() => {
return {
convert: jest.fn().mockImplementation((v) => {
return new Date(v).toUTCString();
}),
};
deserialize: jest.fn().mockImplementation(({ id }: { id: string }) => {
if (id === 'date') {
return {
convert: jest.fn().mockImplementation((v) => {
return new Date(v).toUTCString();
}),
};
}
if (id === 'bytes') {
return {
convert: jest.fn().mockImplementation((v) => {
return `${Math.round(v / 1000)}KB`;
}),
};
}
}),
});
}) as jest.Mocked<FieldFormatsRegistryProvider>;
@ -358,6 +371,10 @@ describe('JobsHealthService', () => {
job_id: 'test_job_01',
log_time: 'Thu, 22 Jul 2021 06:38:34 GMT',
memory_status: 'hard_limit',
model_bytes: '1000KB',
model_bytes_exceeded: '200KB',
model_bytes_memory_limit: '800KB',
peak_model_bytes: '1000KB',
},
],
message:

26
x-pack/plugins/ml/server/lib/alerts/jobs_health_service.ts
View file

@ -55,10 +55,14 @@ export function jobsHealthServiceProvider(
/**
* Provides a callback for date formatting based on the Kibana settings.
*/
const getDateFormatter = memoize(async () => {
const getFormatters = memoize(async () => {
const fieldFormatsRegistry = await getFieldsFormatRegistry();
const dateFormatter = fieldFormatsRegistry.deserialize({ id: 'date' });
return dateFormatter.convert.bind(dateFormatter);
const bytesFormatter = fieldFormatsRegistry.deserialize({ id: 'bytes' });
return {
dateFormatter: dateFormatter.convert.bind(dateFormatter),
bytesFormatter: bytesFormatter.convert.bind(bytesFormatter),
};
});
/**
@ -186,7 +190,7 @@ export function jobsHealthServiceProvider(
async getMmlReport(jobIds: string[]): Promise<MmlTestResponse[]> {
const jobsStats = await getJobStats(jobIds);
const dateFormatter = await getDateFormatter();
const { dateFormatter, bytesFormatter } = await getFormatters();
return jobsStats
.filter((j) => j.state === 'opened' && j.model_size_stats.memory_status !== 'ok')
@ -195,10 +199,10 @@ export function jobsHealthServiceProvider(
job_id: jobId,
memory_status: modelSizeStats.memory_status,
log_time: dateFormatter(modelSizeStats.log_time),
model_bytes: modelSizeStats.model_bytes,
model_bytes_memory_limit: modelSizeStats.model_bytes_memory_limit,
peak_model_bytes: modelSizeStats.peak_model_bytes,
model_bytes_exceeded: modelSizeStats.model_bytes_exceeded,
model_bytes: bytesFormatter(modelSizeStats.model_bytes),
model_bytes_memory_limit: bytesFormatter(modelSizeStats.model_bytes_memory_limit),
peak_model_bytes: bytesFormatter(modelSizeStats.peak_model_bytes),
model_bytes_exceeded: bytesFormatter(modelSizeStats.model_bytes_exceeded),
};
});
},
@ -227,7 +231,7 @@ export function jobsHealthServiceProvider(
const defaultLookbackInterval = resolveLookbackInterval(resultJobs, datafeeds!);
const earliestMs = getDelayedDataLookbackTimestamp(timeInterval, defaultLookbackInterval);
const getFormattedDate = await getDateFormatter();
const { dateFormatter } = await getFormatters();
return (
await annotationService.getDelayedDataAnnotations({
@ -265,7 +269,7 @@ export function jobsHealthServiceProvider(
.map((v) => {
return {
...v,
end_timestamp: getFormattedDate(v.end_timestamp),
end_timestamp: dateFormatter(v.end_timestamp),
};
});
},
@ -279,7 +283,7 @@ export function jobsHealthServiceProvider(
jobIds: string[],
previousStartedAt: Date
): Promise<JobsErrorsResponse[]> {
const getFormattedDate = await getDateFormatter();
const { dateFormatter } = await getFormatters();
return (
await jobAuditMessagesService.getJobsErrorMessages(jobIds, previousStartedAt.getTime())
@ -289,7 +293,7 @@ export function jobsHealthServiceProvider(
errors: v.errors.map((e) => {
return {
...e,
timestamp: getFormattedDate(e.timestamp),
timestamp: dateFormatter(e.timestamp),
};
}),
};

8
x-pack/plugins/ml/server/lib/alerts/register_jobs_monitoring_rule_type.ts
View file

@ -31,10 +31,10 @@ export interface MmlTestResponse {
job_id: string;
memory_status: ModelSizeStats['memory_status'];
log_time: ModelSizeStats['log_time'];
model_bytes: ModelSizeStats['model_bytes'];
model_bytes_memory_limit: ModelSizeStats['model_bytes_memory_limit'];
peak_model_bytes: ModelSizeStats['peak_model_bytes'];
model_bytes_exceeded: ModelSizeStats['model_bytes_exceeded'];
model_bytes: string;
model_bytes_memory_limit: string;
peak_model_bytes: string;
model_bytes_exceeded: string;
}
export interface NotStartedDatafeedResponse {