[ML] Adjusting module jobs model memory limit (#45502)
This commit is contained in:
parent
e0fdce6356
commit
b5fc70e013
|
@ -8,6 +8,8 @@
|
|||
|
||||
import fs from 'fs';
|
||||
import Boom from 'boom';
|
||||
import numeral from '@elastic/numeral';
|
||||
import { idx } from '@kbn/elastic-idx';
|
||||
import { merge } from 'lodash';
|
||||
import { getLatestDataOrBucketTimestamp, prefixDatafeedId } from '../../../common/util/job_utils';
|
||||
import { mlLog } from '../../client/log';
|
||||
|
@ -308,6 +310,7 @@ export class DataRecognizer {
|
|||
this.applyDatafeedConfigOverrides(moduleConfig, datafeedOverrides, jobPrefix);
|
||||
this.updateDatafeedIndices(moduleConfig);
|
||||
this.updateJobUrlIndexPatterns(moduleConfig);
|
||||
await this.updateModelMemoryLimits(moduleConfig);
|
||||
|
||||
// create the jobs
|
||||
if (moduleConfig.jobs && moduleConfig.jobs.length) {
|
||||
|
@ -768,6 +771,30 @@ export class DataRecognizer {
|
|||
}
|
||||
}
|
||||
|
||||
// ensure the model memory limit for each job is not greater than
|
||||
// the max model memory setting for the cluster
|
||||
async updateModelMemoryLimits(moduleConfig) {
|
||||
const { limits } = await this.callWithRequest('ml.info');
|
||||
const maxMml = limits.max_model_memory_limit;
|
||||
if (maxMml !== undefined) {
|
||||
const maxBytes = numeral(maxMml.toUpperCase()).value();
|
||||
|
||||
if (Array.isArray(moduleConfig.jobs)) {
|
||||
moduleConfig.jobs.forEach((job) => {
|
||||
const mml = idx(job, _ => _.config.analysis_limits.model_memory_limit);
|
||||
if (mml !== undefined) {
|
||||
const mmlBytes = numeral(mml.toUpperCase()).value();
|
||||
if (mmlBytes > maxBytes) {
|
||||
// if the job's mml is over the max,
|
||||
// so set the jobs mml to be the max
|
||||
job.config.analysis_limits.model_memory_limit = maxMml;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// check the kibana saved searches JSON in the module to see if they contain INDEX_PATTERN_ID
|
||||
// which needs replacement
|
||||
doSavedObjectsContainIndexPatternId(moduleConfig) {
|
||||
|
|
Loading…
Reference in a new issue