Revert "[Metrics UI] Add ability to override datafeeds and job config for partition field (#78875)"
This reverts commit ee7672aaf0.
This commit is contained in:
Jonathan Budzenski
parent
fd7dd41617
commit
085f8a17ff
|
|
@ -33,11 +33,11 @@ export interface ModuleDescriptor<JobType extends string> {
|
|||
partitionField?: string
|
||||
) => Promise<SetupMlModuleResponsePayload>;
|
||||
cleanUpModule: (spaceId: string, sourceId: string) => Promise<DeleteJobsResponsePayload>;
|
||||
validateSetupIndices?: (
|
||||
validateSetupIndices: (
|
||||
indices: string[],
|
||||
timestampField: string
|
||||
) => Promise<ValidationIndicesResponsePayload>;
|
||||
validateSetupDatasets?: (
|
||||
validateSetupDatasets: (
|
||||
indices: string[],
|
||||
timestampField: string,
|
||||
startTime: number,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,289 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { isEqual } from 'lodash';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { usePrevious } from 'react-use';
|
||||
import {
|
||||
combineDatasetFilters,
|
||||
DatasetFilter,
|
||||
filterDatasetFilter,
|
||||
isExampleDataIndex,
|
||||
} from '../../../common/infra_ml';
|
||||
import {
|
||||
AvailableIndex,
|
||||
ValidationIndicesError,
|
||||
ValidationUIError,
|
||||
} from '../../components/logging/log_analysis_setup/initial_configuration_step';
|
||||
import { useTrackedPromise } from '../../utils/use_tracked_promise';
|
||||
import { ModuleDescriptor, ModuleSourceConfiguration } from './infra_ml_module_types';
|
||||
|
||||
type SetupHandler = (
|
||||
indices: string[],
|
||||
startTime: number | undefined,
|
||||
endTime: number | undefined,
|
||||
datasetFilter: DatasetFilter
|
||||
) => void;
|
||||
|
||||
interface AnalysisSetupStateArguments<JobType extends string> {
|
||||
cleanUpAndSetUpModule: SetupHandler;
|
||||
moduleDescriptor: ModuleDescriptor<JobType>;
|
||||
setUpModule: SetupHandler;
|
||||
sourceConfiguration: ModuleSourceConfiguration;
|
||||
}
|
||||
|
||||
const fourWeeksInMs = 86400000 * 7 * 4;
|
||||
|
||||
export const useAnalysisSetupState = <JobType extends string>({
|
||||
cleanUpAndSetUpModule,
|
||||
moduleDescriptor: { validateSetupDatasets, validateSetupIndices },
|
||||
setUpModule,
|
||||
sourceConfiguration,
|
||||
}: AnalysisSetupStateArguments<JobType>) => {
|
||||
const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
|
||||
const [endTime, setEndTime] = useState<number | undefined>(undefined);
|
||||
|
||||
const isTimeRangeValid = useMemo(
|
||||
() => (startTime != null && endTime != null ? startTime < endTime : true),
|
||||
[endTime, startTime]
|
||||
);
|
||||
|
||||
const [validatedIndices, setValidatedIndices] = useState<AvailableIndex[]>(
|
||||
sourceConfiguration.indices.map((indexName) => ({
|
||||
name: indexName,
|
||||
validity: 'unknown' as const,
|
||||
}))
|
||||
);
|
||||
|
||||
const updateIndicesWithValidationErrors = useCallback(
|
||||
(validationErrors: ValidationIndicesError[]) =>
|
||||
setValidatedIndices((availableIndices) =>
|
||||
availableIndices.map((previousAvailableIndex) => {
|
||||
const indexValiationErrors = validationErrors.filter(
|
||||
({ index }) => index === previousAvailableIndex.name
|
||||
);
|
||||
|
||||
if (indexValiationErrors.length > 0) {
|
||||
return {
|
||||
validity: 'invalid',
|
||||
name: previousAvailableIndex.name,
|
||||
errors: indexValiationErrors,
|
||||
};
|
||||
} else if (previousAvailableIndex.validity === 'valid') {
|
||||
return {
|
||||
...previousAvailableIndex,
|
||||
validity: 'valid',
|
||||
errors: [],
|
||||
};
|
||||
} else {
|
||||
return {
|
||||
validity: 'valid',
|
||||
name: previousAvailableIndex.name,
|
||||
isSelected: !isExampleDataIndex(previousAvailableIndex.name),
|
||||
availableDatasets: [],
|
||||
datasetFilter: {
|
||||
type: 'includeAll' as const,
|
||||
},
|
||||
};
|
||||
}
|
||||
})
|
||||
),
|
||||
[]
|
||||
);
|
||||
|
||||
const updateIndicesWithAvailableDatasets = useCallback(
|
||||
(availableDatasets: Array<{ indexName: string; datasets: string[] }>) =>
|
||||
setValidatedIndices((availableIndices) =>
|
||||
availableIndices.map((previousAvailableIndex) => {
|
||||
if (previousAvailableIndex.validity !== 'valid') {
|
||||
return previousAvailableIndex;
|
||||
}
|
||||
|
||||
const availableDatasetsForIndex = availableDatasets.filter(
|
||||
({ indexName }) => indexName === previousAvailableIndex.name
|
||||
);
|
||||
const newAvailableDatasets = availableDatasetsForIndex.flatMap(
|
||||
({ datasets }) => datasets
|
||||
);
|
||||
|
||||
// filter out datasets that have disappeared if this index' datasets were updated
|
||||
const newDatasetFilter: DatasetFilter =
|
||||
availableDatasetsForIndex.length > 0
|
||||
? filterDatasetFilter(previousAvailableIndex.datasetFilter, (dataset) =>
|
||||
newAvailableDatasets.includes(dataset)
|
||||
)
|
||||
: previousAvailableIndex.datasetFilter;
|
||||
|
||||
return {
|
||||
...previousAvailableIndex,
|
||||
availableDatasets: newAvailableDatasets,
|
||||
datasetFilter: newDatasetFilter,
|
||||
};
|
||||
})
|
||||
),
|
||||
[]
|
||||
);
|
||||
|
||||
const validIndexNames = useMemo(
|
||||
() => validatedIndices.filter((index) => index.validity === 'valid').map((index) => index.name),
|
||||
[validatedIndices]
|
||||
);
|
||||
|
||||
const selectedIndexNames = useMemo(
|
||||
() =>
|
||||
validatedIndices
|
||||
.filter((index) => index.validity === 'valid' && index.isSelected)
|
||||
.map((i) => i.name),
|
||||
[validatedIndices]
|
||||
);
|
||||
|
||||
const datasetFilter = useMemo(
|
||||
() =>
|
||||
validatedIndices
|
||||
.flatMap((validatedIndex) =>
|
||||
validatedIndex.validity === 'valid'
|
||||
? validatedIndex.datasetFilter
|
||||
: { type: 'includeAll' as const }
|
||||
)
|
||||
.reduce(combineDatasetFilters, { type: 'includeAll' as const }),
|
||||
[validatedIndices]
|
||||
);
|
||||
|
||||
const [validateIndicesRequest, validateIndices] = useTrackedPromise(
|
||||
{
|
||||
cancelPreviousOn: 'resolution',
|
||||
createPromise: async () => {
|
||||
return await validateSetupIndices(
|
||||
sourceConfiguration.indices,
|
||||
sourceConfiguration.timestampField
|
||||
);
|
||||
},
|
||||
onResolve: ({ data: { errors } }) => {
|
||||
updateIndicesWithValidationErrors(errors);
|
||||
},
|
||||
onReject: () => {
|
||||
setValidatedIndices([]);
|
||||
},
|
||||
},
|
||||
[sourceConfiguration.indices, sourceConfiguration.timestampField]
|
||||
);
|
||||
|
||||
const [validateDatasetsRequest, validateDatasets] = useTrackedPromise(
|
||||
{
|
||||
cancelPreviousOn: 'resolution',
|
||||
createPromise: async () => {
|
||||
if (validIndexNames.length === 0) {
|
||||
return { data: { datasets: [] } };
|
||||
}
|
||||
|
||||
return await validateSetupDatasets(
|
||||
validIndexNames,
|
||||
sourceConfiguration.timestampField,
|
||||
startTime ?? 0,
|
||||
endTime ?? Date.now()
|
||||
);
|
||||
},
|
||||
onResolve: ({ data: { datasets } }) => {
|
||||
updateIndicesWithAvailableDatasets(datasets);
|
||||
},
|
||||
},
|
||||
[validIndexNames, sourceConfiguration.timestampField, startTime, endTime]
|
||||
);
|
||||
|
||||
const setUp = useCallback(() => {
|
||||
return setUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
|
||||
}, [setUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);
|
||||
|
||||
const cleanUpAndSetUp = useCallback(() => {
|
||||
return cleanUpAndSetUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
|
||||
}, [cleanUpAndSetUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);
|
||||
|
||||
const isValidating = useMemo(
|
||||
() => validateIndicesRequest.state === 'pending' || validateDatasetsRequest.state === 'pending',
|
||||
[validateDatasetsRequest.state, validateIndicesRequest.state]
|
||||
);
|
||||
|
||||
const validationErrors = useMemo<ValidationUIError[]>(() => {
|
||||
if (isValidating) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
// validate request status
|
||||
...(validateIndicesRequest.state === 'rejected' ||
|
||||
validateDatasetsRequest.state === 'rejected'
|
||||
? [{ error: 'NETWORK_ERROR' as const }]
|
||||
: []),
|
||||
// validation request results
|
||||
...validatedIndices.reduce<ValidationUIError[]>((errors, index) => {
|
||||
return index.validity === 'invalid' && selectedIndexNames.includes(index.name)
|
||||
? [...errors, ...index.errors]
|
||||
: errors;
|
||||
}, []),
|
||||
// index count
|
||||
...(selectedIndexNames.length === 0 ? [{ error: 'TOO_FEW_SELECTED_INDICES' as const }] : []),
|
||||
// time range
|
||||
...(!isTimeRangeValid ? [{ error: 'INVALID_TIME_RANGE' as const }] : []),
|
||||
];
|
||||
}, [
|
||||
isValidating,
|
||||
validateIndicesRequest.state,
|
||||
validateDatasetsRequest.state,
|
||||
validatedIndices,
|
||||
selectedIndexNames,
|
||||
isTimeRangeValid,
|
||||
]);
|
||||
|
||||
const prevStartTime = usePrevious(startTime);
|
||||
const prevEndTime = usePrevious(endTime);
|
||||
const prevValidIndexNames = usePrevious(validIndexNames);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isTimeRangeValid) {
|
||||
return;
|
||||
}
|
||||
|
||||
validateIndices();
|
||||
}, [isTimeRangeValid, validateIndices]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isTimeRangeValid) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
startTime !== prevStartTime ||
|
||||
endTime !== prevEndTime ||
|
||||
!isEqual(validIndexNames, prevValidIndexNames)
|
||||
) {
|
||||
validateDatasets();
|
||||
}
|
||||
}, [
|
||||
endTime,
|
||||
isTimeRangeValid,
|
||||
prevEndTime,
|
||||
prevStartTime,
|
||||
prevValidIndexNames,
|
||||
startTime,
|
||||
validIndexNames,
|
||||
validateDatasets,
|
||||
]);
|
||||
|
||||
return {
|
||||
cleanUpAndSetUp,
|
||||
datasetFilter,
|
||||
endTime,
|
||||
isValidating,
|
||||
selectedIndexNames,
|
||||
setEndTime,
|
||||
setStartTime,
|
||||
setUp,
|
||||
startTime,
|
||||
validatedIndices,
|
||||
setValidatedIndices,
|
||||
validationErrors,
|
||||
};
|
||||
};
|
||||
|
|
@ -10,27 +10,17 @@ import { cleanUpJobsAndDatafeeds } from '../../infra_ml_cleanup';
|
|||
import { callJobsSummaryAPI } from '../../api/ml_get_jobs_summary_api';
|
||||
import { callGetMlModuleAPI } from '../../api/ml_get_module';
|
||||
import { callSetupMlModuleAPI } from '../../api/ml_setup_module_api';
|
||||
import { callValidateIndicesAPI } from '../../../logs/log_analysis/api/validate_indices';
|
||||
import { callValidateDatasetsAPI } from '../../../logs/log_analysis/api/validate_datasets';
|
||||
import {
|
||||
metricsHostsJobTypes,
|
||||
getJobId,
|
||||
MetricsHostsJobType,
|
||||
DatasetFilter,
|
||||
bucketSpan,
|
||||
partitionField,
|
||||
} from '../../../../../common/infra_ml';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import MemoryJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_memory_usage.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import MemoryDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_memory_usage.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkInJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_in.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkInDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_in.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkOutJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/hosts_network_out.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkOutDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_hosts/ml/datafeed_hosts_network_out.json';
|
||||
|
||||
type JobType = 'hosts_memory_usage' | 'hosts_network_in' | 'hosts_network_out';
|
||||
const moduleId = 'metrics_ui_hosts';
|
||||
const moduleName = i18n.translate('xpack.infra.ml.metricsModuleName', {
|
||||
defaultMessage: 'Metrics anomanly detection',
|
||||
|
|
@ -64,68 +54,23 @@ const setUpModule = async (
|
|||
end: number | undefined,
|
||||
datasetFilter: DatasetFilter,
|
||||
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration,
|
||||
partitionField?: string
|
||||
pField?: string
|
||||
) => {
|
||||
const indexNamePattern = indices.join(',');
|
||||
const jobIds: JobType[] = ['hosts_memory_usage', 'hosts_network_in', 'hosts_network_out'];
|
||||
|
||||
const jobOverrides = jobIds.map((id) => {
|
||||
const { job: defaultJobConfig } = getDefaultJobConfigs(id);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const analysis_config = {
|
||||
...defaultJobConfig.analysis_config,
|
||||
};
|
||||
|
||||
if (partitionField) {
|
||||
analysis_config.detectors[0].partition_field_name = partitionField;
|
||||
if (analysis_config.influencers.indexOf(partitionField) === -1) {
|
||||
analysis_config.influencers.push(partitionField);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
job_id: id,
|
||||
data_description: {
|
||||
time_field: timestampField,
|
||||
const jobIds = ['hosts_memory_usage', 'hosts_network_in', 'hosts_network_out'];
|
||||
const jobOverrides = jobIds.map((id) => ({
|
||||
job_id: id,
|
||||
data_description: {
|
||||
time_field: timestampField,
|
||||
},
|
||||
custom_settings: {
|
||||
metrics_source_config: {
|
||||
indexPattern: indexNamePattern,
|
||||
timestampField,
|
||||
bucketSpan,
|
||||
},
|
||||
analysis_config,
|
||||
custom_settings: {
|
||||
metrics_source_config: {
|
||||
indexPattern: indexNamePattern,
|
||||
timestampField,
|
||||
bucketSpan,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const datafeedOverrides = jobIds.map((id) => {
|
||||
const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id);
|
||||
|
||||
if (!partitionField || id === 'hosts_memory_usage') {
|
||||
// Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field
|
||||
return defaultDatafeedConfig;
|
||||
}
|
||||
|
||||
// If we have a partition field, we need to change the aggregation to do a terms agg at the top level
|
||||
const aggregations = {
|
||||
[partitionField]: {
|
||||
terms: {
|
||||
field: partitionField,
|
||||
},
|
||||
aggregations: {
|
||||
...defaultDatafeedConfig.aggregations,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...defaultDatafeedConfig,
|
||||
job_id: id,
|
||||
aggregations,
|
||||
};
|
||||
});
|
||||
},
|
||||
}));
|
||||
|
||||
return callSetupMlModuleAPI(
|
||||
moduleId,
|
||||
|
|
@ -135,34 +80,36 @@ const setUpModule = async (
|
|||
sourceId,
|
||||
indexNamePattern,
|
||||
jobOverrides,
|
||||
datafeedOverrides
|
||||
[]
|
||||
);
|
||||
};
|
||||
|
||||
const getDefaultJobConfigs = (jobId: JobType) => {
|
||||
switch (jobId) {
|
||||
case 'hosts_memory_usage':
|
||||
return {
|
||||
datafeed: MemoryDatafeed,
|
||||
job: MemoryJob,
|
||||
};
|
||||
case 'hosts_network_in':
|
||||
return {
|
||||
datafeed: NetworkInDatafeed,
|
||||
job: NetworkInJob,
|
||||
};
|
||||
case 'hosts_network_out':
|
||||
return {
|
||||
datafeed: NetworkOutDatafeed,
|
||||
job: NetworkOutJob,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const cleanUpModule = async (spaceId: string, sourceId: string) => {
|
||||
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, metricsHostsJobTypes);
|
||||
};
|
||||
|
||||
const validateSetupIndices = async (indices: string[], timestampField: string) => {
|
||||
return await callValidateIndicesAPI(indices, [
|
||||
{
|
||||
name: timestampField,
|
||||
validTypes: ['date'],
|
||||
},
|
||||
{
|
||||
name: partitionField,
|
||||
validTypes: ['keyword'],
|
||||
},
|
||||
]);
|
||||
};
|
||||
|
||||
const validateSetupDatasets = async (
|
||||
indices: string[],
|
||||
timestampField: string,
|
||||
startTime: number,
|
||||
endTime: number
|
||||
) => {
|
||||
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
|
||||
};
|
||||
|
||||
export const metricHostsModule: ModuleDescriptor<MetricsHostsJobType> = {
|
||||
moduleId,
|
||||
moduleName,
|
||||
|
|
@ -174,4 +121,6 @@ export const metricHostsModule: ModuleDescriptor<MetricsHostsJobType> = {
|
|||
getModuleDefinition,
|
||||
setUpModule,
|
||||
cleanUpModule,
|
||||
validateSetupDatasets,
|
||||
validateSetupIndices,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -10,28 +10,17 @@ import { cleanUpJobsAndDatafeeds } from '../../infra_ml_cleanup';
|
|||
import { callJobsSummaryAPI } from '../../api/ml_get_jobs_summary_api';
|
||||
import { callGetMlModuleAPI } from '../../api/ml_get_module';
|
||||
import { callSetupMlModuleAPI } from '../../api/ml_setup_module_api';
|
||||
import { callValidateIndicesAPI } from '../../../logs/log_analysis/api/validate_indices';
|
||||
import { callValidateDatasetsAPI } from '../../../logs/log_analysis/api/validate_datasets';
|
||||
import {
|
||||
metricsK8SJobTypes,
|
||||
getJobId,
|
||||
MetricK8sJobType,
|
||||
DatasetFilter,
|
||||
bucketSpan,
|
||||
partitionField,
|
||||
} from '../../../../../common/infra_ml';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import MemoryJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_memory_usage.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import MemoryDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_memory_usage.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkInJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_in.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkInDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_in.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkOutJob from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/k8s_network_out.json';
|
||||
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||
import NetworkOutDatafeed from '../../../../../../ml/server/models/data_recognizer/modules/metrics_ui_k8s/ml/datafeed_k8s_network_out.json';
|
||||
|
||||
type JobType = 'k8s_memory_usage' | 'k8s_network_in' | 'k8s_network_out';
|
||||
export const DEFAULT_K8S_PARTITION_FIELD = 'kubernetes.namespace';
|
||||
const moduleId = 'metrics_ui_k8s';
|
||||
const moduleName = i18n.translate('xpack.infra.ml.metricsModuleName', {
|
||||
defaultMessage: 'Metrics anomanly detection',
|
||||
|
|
@ -65,72 +54,26 @@ const setUpModule = async (
|
|||
end: number | undefined,
|
||||
datasetFilter: DatasetFilter,
|
||||
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration,
|
||||
partitionField?: string
|
||||
pField?: string
|
||||
) => {
|
||||
const indexNamePattern = indices.join(',');
|
||||
const jobIds: JobType[] = ['k8s_memory_usage', 'k8s_network_in', 'k8s_network_out'];
|
||||
const jobOverrides = jobIds.map((id) => {
|
||||
const { job: defaultJobConfig } = getDefaultJobConfigs(id);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const analysis_config = {
|
||||
...defaultJobConfig.analysis_config,
|
||||
};
|
||||
|
||||
if (partitionField) {
|
||||
analysis_config.detectors[0].partition_field_name = partitionField;
|
||||
if (analysis_config.influencers.indexOf(partitionField) === -1) {
|
||||
analysis_config.influencers.push(partitionField);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
job_id: id,
|
||||
data_description: {
|
||||
time_field: timestampField,
|
||||
const jobIds = ['k8s_memory_usage', 'k8s_network_in', 'k8s_network_out'];
|
||||
const jobOverrides = jobIds.map((id) => ({
|
||||
job_id: id,
|
||||
analysis_config: {
|
||||
bucket_span: `${bucketSpan}ms`,
|
||||
},
|
||||
data_description: {
|
||||
time_field: timestampField,
|
||||
},
|
||||
custom_settings: {
|
||||
metrics_source_config: {
|
||||
indexPattern: indexNamePattern,
|
||||
timestampField,
|
||||
bucketSpan,
|
||||
},
|
||||
analysis_config,
|
||||
custom_settings: {
|
||||
metrics_source_config: {
|
||||
indexPattern: indexNamePattern,
|
||||
timestampField,
|
||||
bucketSpan,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const datafeedOverrides = jobIds.map((id) => {
|
||||
const { datafeed: defaultDatafeedConfig } = getDefaultJobConfigs(id);
|
||||
|
||||
if (!partitionField || id === 'k8s_memory_usage') {
|
||||
// Since the host memory usage doesn't have custom aggs, we don't need to do anything to add a partition field
|
||||
return defaultDatafeedConfig;
|
||||
}
|
||||
|
||||
// Because the ML K8s jobs ship with a default partition field of {kubernetes.namespace}, ignore that agg and wrap it in our own agg.
|
||||
const innerAggregation =
|
||||
defaultDatafeedConfig.aggregations[DEFAULT_K8S_PARTITION_FIELD].aggregations;
|
||||
|
||||
// If we have a partition field, we need to change the aggregation to do a terms agg to partition the data at the top level
|
||||
const aggregations = {
|
||||
[partitionField]: {
|
||||
terms: {
|
||||
field: partitionField,
|
||||
size: 25, // 25 is arbitratry and only used to keep the number of buckets to a managable level in the event that the user choose a high cardinality partition field.
|
||||
},
|
||||
aggregations: {
|
||||
...innerAggregation,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
...defaultDatafeedConfig,
|
||||
job_id: id,
|
||||
aggregations,
|
||||
};
|
||||
});
|
||||
},
|
||||
}));
|
||||
|
||||
return callSetupMlModuleAPI(
|
||||
moduleId,
|
||||
|
|
@ -140,34 +83,36 @@ const setUpModule = async (
|
|||
sourceId,
|
||||
indexNamePattern,
|
||||
jobOverrides,
|
||||
datafeedOverrides
|
||||
[]
|
||||
);
|
||||
};
|
||||
|
||||
const getDefaultJobConfigs = (jobId: JobType) => {
|
||||
switch (jobId) {
|
||||
case 'k8s_memory_usage':
|
||||
return {
|
||||
datafeed: MemoryDatafeed,
|
||||
job: MemoryJob,
|
||||
};
|
||||
case 'k8s_network_in':
|
||||
return {
|
||||
datafeed: NetworkInDatafeed,
|
||||
job: NetworkInJob,
|
||||
};
|
||||
case 'k8s_network_out':
|
||||
return {
|
||||
datafeed: NetworkOutDatafeed,
|
||||
job: NetworkOutJob,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const cleanUpModule = async (spaceId: string, sourceId: string) => {
|
||||
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, metricsK8SJobTypes);
|
||||
};
|
||||
|
||||
const validateSetupIndices = async (indices: string[], timestampField: string) => {
|
||||
return await callValidateIndicesAPI(indices, [
|
||||
{
|
||||
name: timestampField,
|
||||
validTypes: ['date'],
|
||||
},
|
||||
{
|
||||
name: partitionField,
|
||||
validTypes: ['keyword'],
|
||||
},
|
||||
]);
|
||||
};
|
||||
|
||||
const validateSetupDatasets = async (
|
||||
indices: string[],
|
||||
timestampField: string,
|
||||
startTime: number,
|
||||
endTime: number
|
||||
) => {
|
||||
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
|
||||
};
|
||||
|
||||
export const metricHostsModule: ModuleDescriptor<MetricK8sJobType> = {
|
||||
moduleId,
|
||||
moduleName,
|
||||
|
|
@ -179,4 +124,6 @@ export const metricHostsModule: ModuleDescriptor<MetricK8sJobType> = {
|
|||
getModuleDefinition,
|
||||
setUpModule,
|
||||
cleanUpModule,
|
||||
validateSetupDatasets,
|
||||
validateSetupIndices,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ export const AnomalyDetectionFlyout = () => {
|
|||
|
||||
return (
|
||||
<>
|
||||
<EuiButtonEmpty iconSide={'left'} iconType={'inspect'} onClick={openFlyout}>
|
||||
<EuiButtonEmpty iconSide={'right'} onClick={openFlyout}>
|
||||
<FormattedMessage
|
||||
id="xpack.infra.ml.anomalyDetectionButton"
|
||||
defaultMessage="Anomaly detection"
|
||||
defaultMessage="Anomaly Detection"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
{showFlyout && (
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
*/
|
||||
|
||||
import React, { useState, useCallback, useEffect } from 'react';
|
||||
import { EuiFlyoutHeader, EuiTitle, EuiFlyoutBody, EuiSpacer } from '@elastic/eui';
|
||||
import { EuiFlyoutHeader, EuiTitle, EuiFlyoutBody, EuiTabs, EuiTab, EuiSpacer } from '@elastic/eui';
|
||||
import { FormattedMessage } from '@kbn/i18n/react';
|
||||
import { EuiText, EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui';
|
||||
import { i18n } from '@kbn/i18n';
|
||||
|
|
@ -30,7 +30,7 @@ interface Props {
|
|||
}
|
||||
|
||||
export const FlyoutHome = (props: Props) => {
|
||||
const [tab] = useState<'jobs' | 'anomalies'>('jobs');
|
||||
const [tab, setTab] = useState<'jobs' | 'anomalies'>('jobs');
|
||||
const { goToSetup } = props;
|
||||
const {
|
||||
fetchJobStatus: fetchHostJobStatus,
|
||||
|
|
@ -56,10 +56,18 @@ export const FlyoutHome = (props: Props) => {
|
|||
goToSetup('kubernetes');
|
||||
}, [goToSetup]);
|
||||
|
||||
const goToJobs = useCallback(() => {
|
||||
setTab('jobs');
|
||||
}, []);
|
||||
|
||||
const jobIds = [
|
||||
...(k8sJobSummaries || []).map((k) => k.id),
|
||||
...(hostJobSummaries || []).map((h) => h.id),
|
||||
];
|
||||
const anomaliesUrl = useLinkProps({
|
||||
app: 'ml',
|
||||
pathname: `/explorer?_g=${createResultsUrl(jobIds)}`,
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
if (hasInfraMLReadCapabilities) {
|
||||
|
|
@ -97,24 +105,30 @@ export const FlyoutHome = (props: Props) => {
|
|||
</EuiFlyoutHeader>
|
||||
|
||||
<EuiFlyoutBody>
|
||||
<div>
|
||||
<EuiText>
|
||||
<p>
|
||||
<FormattedMessage
|
||||
defaultMessage="Anomaly detection is powered by machine learning. Machine learning jobs are available for the following resource types. Enable these jobs to begin detecting anomalies in your infrastructure metrics."
|
||||
id="xpack.infra.ml.anomalyFlyout.create.description"
|
||||
/>
|
||||
</p>
|
||||
</EuiText>
|
||||
</div>
|
||||
|
||||
<EuiTabs>
|
||||
<EuiTab isSelected={tab === 'jobs'} onClick={goToJobs}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.jobsTabLabel"
|
||||
/>
|
||||
</EuiTab>
|
||||
<EuiTab
|
||||
disabled={jobIds.length === 0}
|
||||
isSelected={tab === 'anomalies'}
|
||||
{...anomaliesUrl}
|
||||
>
|
||||
<FormattedMessage
|
||||
defaultMessage="Anomalies"
|
||||
id="xpack.infra.ml.anomalyFlyout.anomaliesTabLabel"
|
||||
/>
|
||||
</EuiTab>
|
||||
</EuiTabs>
|
||||
<EuiSpacer size="l" />
|
||||
{hostJobSummaries.length > 0 && (
|
||||
<>
|
||||
<JobsEnabledCallout
|
||||
hasHostJobs={hostJobSummaries.length > 0}
|
||||
hasK8sJobs={k8sJobSummaries.length > 0}
|
||||
jobIds={jobIds}
|
||||
/>
|
||||
<EuiSpacer size="l" />
|
||||
</>
|
||||
|
|
@ -137,7 +151,6 @@ export const FlyoutHome = (props: Props) => {
|
|||
interface CalloutProps {
|
||||
hasHostJobs: boolean;
|
||||
hasK8sJobs: boolean;
|
||||
jobIds: string[];
|
||||
}
|
||||
const JobsEnabledCallout = (props: CalloutProps) => {
|
||||
let target = '';
|
||||
|
|
@ -162,34 +175,8 @@ const JobsEnabledCallout = (props: CalloutProps) => {
|
|||
pathname: '/jobs',
|
||||
});
|
||||
|
||||
const anomaliesUrl = useLinkProps({
|
||||
app: 'ml',
|
||||
pathname: `/explorer?_g=${createResultsUrl(props.jobIds)}`,
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<EuiFlexGroup gutterSize={'s'}>
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButton {...manageJobsLinkProps} style={{ marginRight: 5 }}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Manage jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.manageJobs"
|
||||
/>
|
||||
</EuiButton>
|
||||
</EuiFlexItem>
|
||||
<EuiFlexItem grow={false}>
|
||||
<EuiButton {...anomaliesUrl}>
|
||||
<FormattedMessage
|
||||
defaultMessage="View anomalies"
|
||||
id="xpack.infra.ml.anomalyFlyout.anomaliesTabLabel"
|
||||
/>
|
||||
</EuiButton>
|
||||
</EuiFlexItem>
|
||||
</EuiFlexGroup>
|
||||
|
||||
<EuiSpacer size="l" />
|
||||
|
||||
<EuiCallOut
|
||||
size="m"
|
||||
color="success"
|
||||
|
|
@ -202,6 +189,13 @@ const JobsEnabledCallout = (props: CalloutProps) => {
|
|||
}
|
||||
iconType="check"
|
||||
/>
|
||||
<EuiSpacer size="l" />
|
||||
<EuiButton {...manageJobsLinkProps}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Manage Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.manageJobs"
|
||||
/>
|
||||
</EuiButton>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
|
@ -217,11 +211,30 @@ interface CreateJobTab {
|
|||
const CreateJobTab = (props: CreateJobTab) => {
|
||||
return (
|
||||
<>
|
||||
{/* <EuiSpacer size="l" /> */}
|
||||
<div>
|
||||
<EuiText>
|
||||
<h3>
|
||||
<FormattedMessage
|
||||
defaultMessage="Create ML Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.jobsTitle"
|
||||
/>
|
||||
</h3>
|
||||
</EuiText>
|
||||
<EuiText>
|
||||
<p>
|
||||
<FormattedMessage
|
||||
defaultMessage="Machine Learning jobs are available for the following resource types. Enable these jobs to begin detecting anomalies in your infrastructure metrics"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.description"
|
||||
/>
|
||||
</p>
|
||||
</EuiText>
|
||||
</div>
|
||||
|
||||
<EuiSpacer size="l" />
|
||||
<EuiFlexGroup gutterSize={'m'}>
|
||||
<EuiFlexItem>
|
||||
<EuiCard
|
||||
isDisabled={!props.hasSetupCapabilities}
|
||||
// isDisabled={props.hasSetupCapabilities}
|
||||
icon={<EuiIcon type={'storage'} />}
|
||||
// title="Hosts"
|
||||
title={
|
||||
|
|
@ -232,7 +245,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
}
|
||||
description={
|
||||
<FormattedMessage
|
||||
defaultMessage="Detect anomalies for memory usage and network traffic."
|
||||
defaultMessage="Detect anomalies for CPU usage, memory usage, network traffic, and load."
|
||||
id="xpack.infra.ml.anomalyFlyout.create.hostDescription"
|
||||
/>
|
||||
}
|
||||
|
|
@ -241,7 +254,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
{props.hasHostJobs && (
|
||||
<EuiButtonEmpty onClick={props.createHosts}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Recreate jobs"
|
||||
defaultMessage="Recreate Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.recreateButton"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
|
|
@ -249,7 +262,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
{!props.hasHostJobs && (
|
||||
<EuiButton onClick={props.createHosts}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Enable"
|
||||
defaultMessage="Create Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.createButton"
|
||||
/>
|
||||
</EuiButton>
|
||||
|
|
@ -260,7 +273,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
</EuiFlexItem>
|
||||
<EuiFlexItem>
|
||||
<EuiCard
|
||||
isDisabled={!props.hasSetupCapabilities}
|
||||
// isDisabled={props.hasSetupCapabilities}
|
||||
icon={<EuiIcon type={'logoKubernetes'} />}
|
||||
title={
|
||||
<FormattedMessage
|
||||
|
|
@ -270,7 +283,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
}
|
||||
description={
|
||||
<FormattedMessage
|
||||
defaultMessage="Detect anomalies for memory usage and network traffic."
|
||||
defaultMessage="Detect anomalies for CPU usage, memory usage, network traffic, and load."
|
||||
id="xpack.infra.ml.anomalyFlyout.create.k8sDescription"
|
||||
/>
|
||||
}
|
||||
|
|
@ -279,7 +292,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
{props.hasK8sJobs && (
|
||||
<EuiButtonEmpty onClick={props.createK8s}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Recreate jobs"
|
||||
defaultMessage="Recreate Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.recreateButton"
|
||||
/>
|
||||
</EuiButtonEmpty>
|
||||
|
|
@ -287,7 +300,7 @@ const CreateJobTab = (props: CreateJobTab) => {
|
|||
{!props.hasK8sJobs && (
|
||||
<EuiButton onClick={props.createK8s}>
|
||||
<FormattedMessage
|
||||
defaultMessage="Enable"
|
||||
defaultMessage="Create Jobs"
|
||||
id="xpack.infra.ml.anomalyFlyout.create.createButton"
|
||||
/>
|
||||
</EuiButton>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,6 @@ import { useSourceViaHttp } from '../../../../../../containers/source/use_source
|
|||
import { useMetricK8sModuleContext } from '../../../../../../containers/ml/modules/metrics_k8s/module';
|
||||
import { useMetricHostsModuleContext } from '../../../../../../containers/ml/modules/metrics_hosts/module';
|
||||
import { FixedDatePicker } from '../../../../../../components/fixed_datepicker';
|
||||
import { DEFAULT_K8S_PARTITION_FIELD } from '../../../../../../containers/ml/modules/metrics_k8s/module_descriptor';
|
||||
|
||||
interface Props {
|
||||
jobType: 'hosts' | 'kubernetes';
|
||||
|
|
@ -108,7 +107,7 @@ export const JobSetupScreen = (props: Props) => {
|
|||
|
||||
useEffect(() => {
|
||||
if (props.jobType === 'kubernetes') {
|
||||
setPartitionField([DEFAULT_K8S_PARTITION_FIELD]);
|
||||
setPartitionField(['kubernetes.namespace']);
|
||||
}
|
||||
}, [props.jobType]);
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue