[Logs UI] Add dataset filter to ML module setup screen (#64470)
This adds the ability to filter the datasets to be processed by the ML jobs on the setup screen.
This commit is contained in:
parent
ccede29e60
commit
39e31d6123
|
@ -0,0 +1,44 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as rt from 'io-ts';
|
||||||
|
|
||||||
|
export const LOG_ANALYSIS_VALIDATE_DATASETS_PATH =
|
||||||
|
'/api/infra/log_analysis/validation/log_entry_datasets';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Request types
|
||||||
|
*/
|
||||||
|
export const validateLogEntryDatasetsRequestPayloadRT = rt.type({
|
||||||
|
data: rt.type({
|
||||||
|
indices: rt.array(rt.string),
|
||||||
|
timestampField: rt.string,
|
||||||
|
startTime: rt.number,
|
||||||
|
endTime: rt.number,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ValidateLogEntryDatasetsRequestPayload = rt.TypeOf<
|
||||||
|
typeof validateLogEntryDatasetsRequestPayloadRT
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Response types
|
||||||
|
* */
|
||||||
|
const logEntryDatasetsEntryRT = rt.strict({
|
||||||
|
indexName: rt.string,
|
||||||
|
datasets: rt.array(rt.string),
|
||||||
|
});
|
||||||
|
|
||||||
|
export const validateLogEntryDatasetsResponsePayloadRT = rt.type({
|
||||||
|
data: rt.type({
|
||||||
|
datasets: rt.array(logEntryDatasetsEntryRT),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ValidateLogEntryDatasetsResponsePayload = rt.TypeOf<
|
||||||
|
typeof validateLogEntryDatasetsResponsePayloadRT
|
||||||
|
>;
|
|
@ -4,4 +4,5 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
export * from './datasets';
|
||||||
export * from './log_entry_rate_indices';
|
export * from './log_entry_rate_indices';
|
||||||
|
|
|
@ -21,17 +21,73 @@ export const getJobId = (spaceId: string, sourceId: string, jobType: string) =>
|
||||||
export const getDatafeedId = (spaceId: string, sourceId: string, jobType: string) =>
|
export const getDatafeedId = (spaceId: string, sourceId: string, jobType: string) =>
|
||||||
`datafeed-${getJobId(spaceId, sourceId, jobType)}`;
|
`datafeed-${getJobId(spaceId, sourceId, jobType)}`;
|
||||||
|
|
||||||
export const jobSourceConfigurationRT = rt.type({
|
export const datasetFilterRT = rt.union([
|
||||||
|
rt.strict({
|
||||||
|
type: rt.literal('includeAll'),
|
||||||
|
}),
|
||||||
|
rt.strict({
|
||||||
|
type: rt.literal('includeSome'),
|
||||||
|
datasets: rt.array(rt.string),
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
export type DatasetFilter = rt.TypeOf<typeof datasetFilterRT>;
|
||||||
|
|
||||||
|
export const jobSourceConfigurationRT = rt.partial({
|
||||||
indexPattern: rt.string,
|
indexPattern: rt.string,
|
||||||
timestampField: rt.string,
|
timestampField: rt.string,
|
||||||
bucketSpan: rt.number,
|
bucketSpan: rt.number,
|
||||||
|
datasetFilter: datasetFilterRT,
|
||||||
});
|
});
|
||||||
|
|
||||||
export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;
|
export type JobSourceConfiguration = rt.TypeOf<typeof jobSourceConfigurationRT>;
|
||||||
|
|
||||||
export const jobCustomSettingsRT = rt.partial({
|
export const jobCustomSettingsRT = rt.partial({
|
||||||
job_revision: rt.number,
|
job_revision: rt.number,
|
||||||
logs_source_config: rt.partial(jobSourceConfigurationRT.props),
|
logs_source_config: jobSourceConfigurationRT,
|
||||||
});
|
});
|
||||||
|
|
||||||
export type JobCustomSettings = rt.TypeOf<typeof jobCustomSettingsRT>;
|
export type JobCustomSettings = rt.TypeOf<typeof jobCustomSettingsRT>;
|
||||||
|
|
||||||
|
export const combineDatasetFilters = (
|
||||||
|
firstFilter: DatasetFilter,
|
||||||
|
secondFilter: DatasetFilter
|
||||||
|
): DatasetFilter => {
|
||||||
|
if (firstFilter.type === 'includeAll' && secondFilter.type === 'includeAll') {
|
||||||
|
return {
|
||||||
|
type: 'includeAll',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const includedDatasets = new Set([
|
||||||
|
...(firstFilter.type === 'includeSome' ? firstFilter.datasets : []),
|
||||||
|
...(secondFilter.type === 'includeSome' ? secondFilter.datasets : []),
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
type: 'includeSome',
|
||||||
|
datasets: [...includedDatasets],
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export const filterDatasetFilter = (
|
||||||
|
datasetFilter: DatasetFilter,
|
||||||
|
predicate: (dataset: string) => boolean
|
||||||
|
): DatasetFilter => {
|
||||||
|
if (datasetFilter.type === 'includeAll') {
|
||||||
|
return datasetFilter;
|
||||||
|
} else {
|
||||||
|
const newDatasets = datasetFilter.datasets.filter(predicate);
|
||||||
|
|
||||||
|
if (newDatasets.length > 0) {
|
||||||
|
return {
|
||||||
|
type: 'includeSome',
|
||||||
|
datasets: newDatasets,
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
type: 'includeAll',
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
|
@ -4,56 +4,41 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { EuiCode, EuiDescribedFormGroup, EuiFormRow, EuiCheckbox, EuiToolTip } from '@elastic/eui';
|
import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
|
||||||
import { i18n } from '@kbn/i18n';
|
import { i18n } from '@kbn/i18n';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React, { useCallback, useMemo } from 'react';
|
import React, { useCallback } from 'react';
|
||||||
|
|
||||||
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
|
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
|
||||||
import { ValidatedIndex, ValidationIndicesUIError } from './validation';
|
import { IndexSetupRow } from './index_setup_row';
|
||||||
|
import { AvailableIndex } from './validation';
|
||||||
|
|
||||||
export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
indices: ValidatedIndex[];
|
indices: AvailableIndex[];
|
||||||
isValidating: boolean;
|
isValidating: boolean;
|
||||||
onChangeSelectedIndices: (selectedIndices: ValidatedIndex[]) => void;
|
onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||||
valid: boolean;
|
valid: boolean;
|
||||||
}> = ({ disabled = false, indices, isValidating, onChangeSelectedIndices, valid }) => {
|
}> = ({ disabled = false, indices, isValidating, onChangeSelectedIndices, valid }) => {
|
||||||
const handleCheckboxChange = useCallback(
|
const changeIsIndexSelected = useCallback(
|
||||||
(event: React.ChangeEvent<HTMLInputElement>) => {
|
(indexName: string, isSelected: boolean) => {
|
||||||
onChangeSelectedIndices(
|
onChangeSelectedIndices(
|
||||||
indices.map(index => {
|
indices.map(index => {
|
||||||
const checkbox = event.currentTarget;
|
return index.name === indexName ? { ...index, isSelected } : index;
|
||||||
return index.name === checkbox.id ? { ...index, isSelected: checkbox.checked } : index;
|
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
[indices, onChangeSelectedIndices]
|
[indices, onChangeSelectedIndices]
|
||||||
);
|
);
|
||||||
|
|
||||||
const choices = useMemo(
|
const changeDatasetFilter = useCallback(
|
||||||
() =>
|
(indexName: string, datasetFilter) => {
|
||||||
indices.map(index => {
|
onChangeSelectedIndices(
|
||||||
const checkbox = (
|
indices.map(index => {
|
||||||
<EuiCheckbox
|
return index.name === indexName ? { ...index, datasetFilter } : index;
|
||||||
key={index.name}
|
})
|
||||||
id={index.name}
|
);
|
||||||
label={<EuiCode>{index.name}</EuiCode>}
|
},
|
||||||
onChange={handleCheckboxChange}
|
[indices, onChangeSelectedIndices]
|
||||||
checked={index.validity === 'valid' && index.isSelected}
|
|
||||||
disabled={disabled || index.validity === 'invalid'}
|
|
||||||
/>
|
|
||||||
);
|
|
||||||
|
|
||||||
return index.validity === 'valid' ? (
|
|
||||||
checkbox
|
|
||||||
) : (
|
|
||||||
<div key={index.name}>
|
|
||||||
<EuiToolTip content={formatValidationError(index.errors)}>{checkbox}</EuiToolTip>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}),
|
|
||||||
[disabled, handleCheckboxChange, indices]
|
|
||||||
);
|
);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
@ -69,13 +54,23 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
||||||
description={
|
description={
|
||||||
<FormattedMessage
|
<FormattedMessage
|
||||||
id="xpack.infra.analysisSetup.indicesSelectionDescription"
|
id="xpack.infra.analysisSetup.indicesSelectionDescription"
|
||||||
defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries."
|
defaultMessage="By default, Machine Learning analyzes log messages in all log indices configured for the source. You can choose to only analyze a subset of the index names. Every selected index name must match at least one index with log entries. You can also choose to only include a certain subset of datasets. Note that the dataset filter applies to all selected indices."
|
||||||
/>
|
/>
|
||||||
}
|
}
|
||||||
>
|
>
|
||||||
<LoadingOverlayWrapper isLoading={isValidating}>
|
<LoadingOverlayWrapper isLoading={isValidating}>
|
||||||
<EuiFormRow fullWidth isInvalid={!valid} label={indicesSelectionLabel} labelType="legend">
|
<EuiFormRow fullWidth isInvalid={!valid} label={indicesSelectionLabel} labelType="legend">
|
||||||
<>{choices}</>
|
<>
|
||||||
|
{indices.map(index => (
|
||||||
|
<IndexSetupRow
|
||||||
|
index={index}
|
||||||
|
isDisabled={disabled}
|
||||||
|
key={index.name}
|
||||||
|
onChangeIsSelected={changeIsIndexSelected}
|
||||||
|
onChangeDatasetFilter={changeDatasetFilter}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
</EuiFormRow>
|
</EuiFormRow>
|
||||||
</LoadingOverlayWrapper>
|
</LoadingOverlayWrapper>
|
||||||
</EuiDescribedFormGroup>
|
</EuiDescribedFormGroup>
|
||||||
|
@ -85,51 +80,3 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
||||||
const indicesSelectionLabel = i18n.translate('xpack.infra.analysisSetup.indicesSelectionLabel', {
|
const indicesSelectionLabel = i18n.translate('xpack.infra.analysisSetup.indicesSelectionLabel', {
|
||||||
defaultMessage: 'Indices',
|
defaultMessage: 'Indices',
|
||||||
});
|
});
|
||||||
|
|
||||||
const formatValidationError = (errors: ValidationIndicesUIError[]): React.ReactNode => {
|
|
||||||
return errors.map(error => {
|
|
||||||
switch (error.error) {
|
|
||||||
case 'INDEX_NOT_FOUND':
|
|
||||||
return (
|
|
||||||
<p key={`${error.error}-${error.index}`}>
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.analysisSetup.indicesSelectionIndexNotFound"
|
|
||||||
defaultMessage="No indices match the pattern {index}"
|
|
||||||
values={{ index: <EuiCode>{error.index}</EuiCode> }}
|
|
||||||
/>
|
|
||||||
</p>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'FIELD_NOT_FOUND':
|
|
||||||
return (
|
|
||||||
<p key={`${error.error}-${error.index}-${error.field}`}>
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.analysisSetup.indicesSelectionNoTimestampField"
|
|
||||||
defaultMessage="At least one index matching {index} lacks a required field {field}."
|
|
||||||
values={{
|
|
||||||
index: <EuiCode>{error.index}</EuiCode>,
|
|
||||||
field: <EuiCode>{error.field}</EuiCode>,
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
</p>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'FIELD_NOT_VALID':
|
|
||||||
return (
|
|
||||||
<p key={`${error.error}-${error.index}-${error.field}`}>
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.analysisSetup.indicesSelectionTimestampNotValid"
|
|
||||||
defaultMessage="At least one index matching {index} has a field called {field} without the correct type."
|
|
||||||
values={{
|
|
||||||
index: <EuiCode>{error.index}</EuiCode>,
|
|
||||||
field: <EuiCode>{error.field}</EuiCode>,
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
</p>
|
|
||||||
);
|
|
||||||
|
|
||||||
default:
|
|
||||||
return '';
|
|
||||||
}
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
|
@ -0,0 +1,88 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
EuiFilterButton,
|
||||||
|
EuiFilterGroup,
|
||||||
|
EuiPopover,
|
||||||
|
EuiPopoverTitle,
|
||||||
|
EuiSelectable,
|
||||||
|
EuiSelectableOption,
|
||||||
|
} from '@elastic/eui';
|
||||||
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
|
import React, { useCallback, useMemo } from 'react';
|
||||||
|
import { DatasetFilter } from '../../../../../common/log_analysis';
|
||||||
|
import { useVisibilityState } from '../../../../utils/use_visibility_state';
|
||||||
|
|
||||||
|
export const IndexSetupDatasetFilter: React.FC<{
|
||||||
|
availableDatasets: string[];
|
||||||
|
datasetFilter: DatasetFilter;
|
||||||
|
isDisabled?: boolean;
|
||||||
|
onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
|
||||||
|
}> = ({ availableDatasets, datasetFilter, isDisabled, onChangeDatasetFilter }) => {
|
||||||
|
const { isVisible, hide, show } = useVisibilityState(false);
|
||||||
|
|
||||||
|
const changeDatasetFilter = useCallback(
|
||||||
|
(options: EuiSelectableOption[]) => {
|
||||||
|
const selectedDatasets = options
|
||||||
|
.filter(({ checked }) => checked === 'on')
|
||||||
|
.map(({ label }) => label);
|
||||||
|
|
||||||
|
onChangeDatasetFilter(
|
||||||
|
selectedDatasets.length === 0
|
||||||
|
? { type: 'includeAll' }
|
||||||
|
: { type: 'includeSome', datasets: selectedDatasets }
|
||||||
|
);
|
||||||
|
},
|
||||||
|
[onChangeDatasetFilter]
|
||||||
|
);
|
||||||
|
|
||||||
|
const selectableOptions: EuiSelectableOption[] = useMemo(
|
||||||
|
() =>
|
||||||
|
availableDatasets.map(datasetName => ({
|
||||||
|
label: datasetName,
|
||||||
|
checked:
|
||||||
|
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName)
|
||||||
|
? 'on'
|
||||||
|
: undefined,
|
||||||
|
})),
|
||||||
|
[availableDatasets, datasetFilter]
|
||||||
|
);
|
||||||
|
|
||||||
|
const datasetFilterButton = (
|
||||||
|
<EuiFilterButton disabled={isDisabled} isSelected={isVisible} onClick={show}>
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.analysisSetup.indexDatasetFilterIncludeAllButtonLabel"
|
||||||
|
defaultMessage="{includeType, select, includeAll {All datasets} includeSome {{includedDatasetCount, plural, one {# dataset} other {# datasets}}}}"
|
||||||
|
values={{
|
||||||
|
includeType: datasetFilter.type,
|
||||||
|
includedDatasetCount:
|
||||||
|
datasetFilter.type === 'includeSome' ? datasetFilter.datasets.length : 0,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</EuiFilterButton>
|
||||||
|
);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<EuiFilterGroup>
|
||||||
|
<EuiPopover
|
||||||
|
button={datasetFilterButton}
|
||||||
|
closePopover={hide}
|
||||||
|
isOpen={isVisible}
|
||||||
|
panelPaddingSize="none"
|
||||||
|
>
|
||||||
|
<EuiSelectable onChange={changeDatasetFilter} options={selectableOptions} searchable>
|
||||||
|
{(list, search) => (
|
||||||
|
<div>
|
||||||
|
<EuiPopoverTitle>{search}</EuiPopoverTitle>
|
||||||
|
{list}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</EuiSelectable>
|
||||||
|
</EuiPopover>
|
||||||
|
</EuiFilterGroup>
|
||||||
|
);
|
||||||
|
};
|
|
@ -0,0 +1,110 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiToolTip } from '@elastic/eui';
|
||||||
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
|
import React, { useCallback } from 'react';
|
||||||
|
import { DatasetFilter } from '../../../../../common/log_analysis';
|
||||||
|
import { IndexSetupDatasetFilter } from './index_setup_dataset_filter';
|
||||||
|
import { AvailableIndex, ValidationIndicesUIError } from './validation';
|
||||||
|
|
||||||
|
export const IndexSetupRow: React.FC<{
|
||||||
|
index: AvailableIndex;
|
||||||
|
isDisabled: boolean;
|
||||||
|
onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void;
|
||||||
|
onChangeIsSelected: (indexName: string, isSelected: boolean) => void;
|
||||||
|
}> = ({ index, isDisabled, onChangeDatasetFilter, onChangeIsSelected }) => {
|
||||||
|
const changeIsSelected = useCallback(
|
||||||
|
(event: React.ChangeEvent<HTMLInputElement>) => {
|
||||||
|
onChangeIsSelected(index.name, event.currentTarget.checked);
|
||||||
|
},
|
||||||
|
[index.name, onChangeIsSelected]
|
||||||
|
);
|
||||||
|
|
||||||
|
const changeDatasetFilter = useCallback(
|
||||||
|
(datasetFilter: DatasetFilter) => onChangeDatasetFilter(index.name, datasetFilter),
|
||||||
|
[index.name, onChangeDatasetFilter]
|
||||||
|
);
|
||||||
|
|
||||||
|
const isSelected = index.validity === 'valid' && index.isSelected;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<EuiFlexGroup alignItems="center">
|
||||||
|
<EuiFlexItem>
|
||||||
|
<EuiCheckbox
|
||||||
|
key={index.name}
|
||||||
|
id={index.name}
|
||||||
|
label={<EuiCode>{index.name}</EuiCode>}
|
||||||
|
onChange={changeIsSelected}
|
||||||
|
checked={isSelected}
|
||||||
|
disabled={isDisabled || index.validity === 'invalid'}
|
||||||
|
/>
|
||||||
|
</EuiFlexItem>
|
||||||
|
<EuiFlexItem grow={false}>
|
||||||
|
{index.validity === 'invalid' ? (
|
||||||
|
<EuiToolTip content={formatValidationError(index.errors)}>
|
||||||
|
<EuiIcon type="alert" color="danger" />
|
||||||
|
</EuiToolTip>
|
||||||
|
) : index.validity === 'valid' ? (
|
||||||
|
<IndexSetupDatasetFilter
|
||||||
|
availableDatasets={index.availableDatasets}
|
||||||
|
datasetFilter={index.datasetFilter}
|
||||||
|
isDisabled={!isSelected || isDisabled}
|
||||||
|
onChangeDatasetFilter={changeDatasetFilter}
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
|
</EuiFlexItem>
|
||||||
|
</EuiFlexGroup>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const formatValidationError = (errors: ValidationIndicesUIError[]): React.ReactNode => {
|
||||||
|
return errors.map(error => {
|
||||||
|
switch (error.error) {
|
||||||
|
case 'INDEX_NOT_FOUND':
|
||||||
|
return (
|
||||||
|
<p key={`${error.error}-${error.index}`}>
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.analysisSetup.indicesSelectionIndexNotFound"
|
||||||
|
defaultMessage="No indices match the pattern {index}"
|
||||||
|
values={{ index: <EuiCode>{error.index}</EuiCode> }}
|
||||||
|
/>
|
||||||
|
</p>
|
||||||
|
);
|
||||||
|
|
||||||
|
case 'FIELD_NOT_FOUND':
|
||||||
|
return (
|
||||||
|
<p key={`${error.error}-${error.index}-${error.field}`}>
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.analysisSetup.indicesSelectionNoTimestampField"
|
||||||
|
defaultMessage="At least one index matching {index} lacks a required field {field}."
|
||||||
|
values={{
|
||||||
|
index: <EuiCode>{error.index}</EuiCode>,
|
||||||
|
field: <EuiCode>{error.field}</EuiCode>,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</p>
|
||||||
|
);
|
||||||
|
|
||||||
|
case 'FIELD_NOT_VALID':
|
||||||
|
return (
|
||||||
|
<p key={`${error.error}-${error.index}-${error.field}`}>
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.analysisSetup.indicesSelectionTimestampNotValid"
|
||||||
|
defaultMessage="At least one index matching {index} has a field called {field} without the correct type."
|
||||||
|
values={{
|
||||||
|
index: <EuiCode>{error.index}</EuiCode>,
|
||||||
|
field: <EuiCode>{error.field}</EuiCode>,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</p>
|
||||||
|
);
|
||||||
|
|
||||||
|
default:
|
||||||
|
return '';
|
||||||
|
}
|
||||||
|
});
|
||||||
|
};
|
|
@ -13,7 +13,7 @@ import React, { useMemo } from 'react';
|
||||||
import { SetupStatus } from '../../../../../common/log_analysis';
|
import { SetupStatus } from '../../../../../common/log_analysis';
|
||||||
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
|
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
|
||||||
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
|
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
|
||||||
import { ValidatedIndex, ValidationIndicesUIError } from './validation';
|
import { AvailableIndex, ValidationIndicesUIError } from './validation';
|
||||||
|
|
||||||
interface InitialConfigurationStepProps {
|
interface InitialConfigurationStepProps {
|
||||||
setStartTime: (startTime: number | undefined) => void;
|
setStartTime: (startTime: number | undefined) => void;
|
||||||
|
@ -21,9 +21,9 @@ interface InitialConfigurationStepProps {
|
||||||
startTime: number | undefined;
|
startTime: number | undefined;
|
||||||
endTime: number | undefined;
|
endTime: number | undefined;
|
||||||
isValidating: boolean;
|
isValidating: boolean;
|
||||||
validatedIndices: ValidatedIndex[];
|
validatedIndices: AvailableIndex[];
|
||||||
setupStatus: SetupStatus;
|
setupStatus: SetupStatus;
|
||||||
setValidatedIndices: (selectedIndices: ValidatedIndex[]) => void;
|
setValidatedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||||
validationErrors?: ValidationIndicesUIError[];
|
validationErrors?: ValidationIndicesUIError[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -5,22 +5,35 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { ValidationIndicesError } from '../../../../../common/http_api';
|
import { ValidationIndicesError } from '../../../../../common/http_api';
|
||||||
|
import { DatasetFilter } from '../../../../../common/log_analysis';
|
||||||
|
|
||||||
|
export { ValidationIndicesError };
|
||||||
|
|
||||||
export type ValidationIndicesUIError =
|
export type ValidationIndicesUIError =
|
||||||
| ValidationIndicesError
|
| ValidationIndicesError
|
||||||
| { error: 'NETWORK_ERROR' }
|
| { error: 'NETWORK_ERROR' }
|
||||||
| { error: 'TOO_FEW_SELECTED_INDICES' };
|
| { error: 'TOO_FEW_SELECTED_INDICES' };
|
||||||
|
|
||||||
interface ValidIndex {
|
interface ValidAvailableIndex {
|
||||||
validity: 'valid';
|
validity: 'valid';
|
||||||
name: string;
|
name: string;
|
||||||
isSelected: boolean;
|
isSelected: boolean;
|
||||||
|
availableDatasets: string[];
|
||||||
|
datasetFilter: DatasetFilter;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface InvalidIndex {
|
interface InvalidAvailableIndex {
|
||||||
validity: 'invalid';
|
validity: 'invalid';
|
||||||
name: string;
|
name: string;
|
||||||
errors: ValidationIndicesError[];
|
errors: ValidationIndicesError[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export type ValidatedIndex = ValidIndex | InvalidIndex;
|
interface UnvalidatedAvailableIndex {
|
||||||
|
validity: 'unknown';
|
||||||
|
name: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type AvailableIndex =
|
||||||
|
| ValidAvailableIndex
|
||||||
|
| InvalidAvailableIndex
|
||||||
|
| UnvalidatedAvailableIndex;
|
||||||
|
|
|
@ -21,7 +21,8 @@ export const callSetupMlModuleAPI = async (
|
||||||
sourceId: string,
|
sourceId: string,
|
||||||
indexPattern: string,
|
indexPattern: string,
|
||||||
jobOverrides: SetupMlModuleJobOverrides[] = [],
|
jobOverrides: SetupMlModuleJobOverrides[] = [],
|
||||||
datafeedOverrides: SetupMlModuleDatafeedOverrides[] = []
|
datafeedOverrides: SetupMlModuleDatafeedOverrides[] = [],
|
||||||
|
query?: object
|
||||||
) => {
|
) => {
|
||||||
const response = await npStart.http.fetch(`/api/ml/modules/setup/${moduleId}`, {
|
const response = await npStart.http.fetch(`/api/ml/modules/setup/${moduleId}`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
@ -34,6 +35,7 @@ export const callSetupMlModuleAPI = async (
|
||||||
startDatafeed: true,
|
startDatafeed: true,
|
||||||
jobOverrides,
|
jobOverrides,
|
||||||
datafeedOverrides,
|
datafeedOverrides,
|
||||||
|
query,
|
||||||
})
|
})
|
||||||
),
|
),
|
||||||
});
|
});
|
||||||
|
@ -60,13 +62,20 @@ const setupMlModuleDatafeedOverridesRT = rt.object;
|
||||||
|
|
||||||
export type SetupMlModuleDatafeedOverrides = rt.TypeOf<typeof setupMlModuleDatafeedOverridesRT>;
|
export type SetupMlModuleDatafeedOverrides = rt.TypeOf<typeof setupMlModuleDatafeedOverridesRT>;
|
||||||
|
|
||||||
const setupMlModuleRequestParamsRT = rt.type({
|
const setupMlModuleRequestParamsRT = rt.intersection([
|
||||||
indexPatternName: rt.string,
|
rt.strict({
|
||||||
prefix: rt.string,
|
indexPatternName: rt.string,
|
||||||
startDatafeed: rt.boolean,
|
prefix: rt.string,
|
||||||
jobOverrides: rt.array(setupMlModuleJobOverridesRT),
|
startDatafeed: rt.boolean,
|
||||||
datafeedOverrides: rt.array(setupMlModuleDatafeedOverridesRT),
|
jobOverrides: rt.array(setupMlModuleJobOverridesRT),
|
||||||
});
|
datafeedOverrides: rt.array(setupMlModuleDatafeedOverridesRT),
|
||||||
|
}),
|
||||||
|
rt.exact(
|
||||||
|
rt.partial({
|
||||||
|
query: rt.object,
|
||||||
|
})
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
|
||||||
const setupMlModuleRequestPayloadRT = rt.intersection([
|
const setupMlModuleRequestPayloadRT = rt.intersection([
|
||||||
setupMlModuleTimeParamsRT,
|
setupMlModuleTimeParamsRT,
|
||||||
|
|
|
@ -0,0 +1,36 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
LOG_ANALYSIS_VALIDATE_DATASETS_PATH,
|
||||||
|
validateLogEntryDatasetsRequestPayloadRT,
|
||||||
|
validateLogEntryDatasetsResponsePayloadRT,
|
||||||
|
} from '../../../../../common/http_api';
|
||||||
|
import { decodeOrThrow } from '../../../../../common/runtime_types';
|
||||||
|
import { npStart } from '../../../../legacy_singletons';
|
||||||
|
|
||||||
|
export const callValidateDatasetsAPI = async (
|
||||||
|
indices: string[],
|
||||||
|
timestampField: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number
|
||||||
|
) => {
|
||||||
|
const response = await npStart.http.fetch(LOG_ANALYSIS_VALIDATE_DATASETS_PATH, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(
|
||||||
|
validateLogEntryDatasetsRequestPayloadRT.encode({
|
||||||
|
data: {
|
||||||
|
endTime,
|
||||||
|
indices,
|
||||||
|
startTime,
|
||||||
|
timestampField,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
return decodeOrThrow(validateLogEntryDatasetsResponsePayloadRT)(response);
|
||||||
|
};
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { useCallback, useMemo } from 'react';
|
import { useCallback, useMemo } from 'react';
|
||||||
|
import { DatasetFilter } from '../../../../common/log_analysis';
|
||||||
import { useTrackedPromise } from '../../../utils/use_tracked_promise';
|
import { useTrackedPromise } from '../../../utils/use_tracked_promise';
|
||||||
import { useModuleStatus } from './log_analysis_module_status';
|
import { useModuleStatus } from './log_analysis_module_status';
|
||||||
import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types';
|
import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types';
|
||||||
|
@ -48,10 +48,11 @@ export const useLogAnalysisModule = <JobType extends string>({
|
||||||
createPromise: async (
|
createPromise: async (
|
||||||
selectedIndices: string[],
|
selectedIndices: string[],
|
||||||
start: number | undefined,
|
start: number | undefined,
|
||||||
end: number | undefined
|
end: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter
|
||||||
) => {
|
) => {
|
||||||
dispatchModuleStatus({ type: 'startedSetup' });
|
dispatchModuleStatus({ type: 'startedSetup' });
|
||||||
const setupResult = await moduleDescriptor.setUpModule(start, end, {
|
const setupResult = await moduleDescriptor.setUpModule(start, end, datasetFilter, {
|
||||||
indices: selectedIndices,
|
indices: selectedIndices,
|
||||||
sourceId,
|
sourceId,
|
||||||
spaceId,
|
spaceId,
|
||||||
|
@ -92,11 +93,16 @@ export const useLogAnalysisModule = <JobType extends string>({
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const cleanUpAndSetUpModule = useCallback(
|
const cleanUpAndSetUpModule = useCallback(
|
||||||
(selectedIndices: string[], start: number | undefined, end: number | undefined) => {
|
(
|
||||||
|
selectedIndices: string[],
|
||||||
|
start: number | undefined,
|
||||||
|
end: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter
|
||||||
|
) => {
|
||||||
dispatchModuleStatus({ type: 'startedSetup' });
|
dispatchModuleStatus({ type: 'startedSetup' });
|
||||||
cleanUpModule()
|
cleanUpModule()
|
||||||
.then(() => {
|
.then(() => {
|
||||||
setUpModule(selectedIndices, start, end);
|
setUpModule(selectedIndices, start, end, datasetFilter);
|
||||||
})
|
})
|
||||||
.catch(() => {
|
.catch(() => {
|
||||||
dispatchModuleStatus({ type: 'failedSetup' });
|
dispatchModuleStatus({ type: 'failedSetup' });
|
||||||
|
|
|
@ -8,7 +8,11 @@ import { DeleteJobsResponsePayload } from './api/ml_cleanup';
|
||||||
import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api';
|
import { FetchJobStatusResponsePayload } from './api/ml_get_jobs_summary_api';
|
||||||
import { GetMlModuleResponsePayload } from './api/ml_get_module';
|
import { GetMlModuleResponsePayload } from './api/ml_get_module';
|
||||||
import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
|
import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
|
||||||
import { ValidationIndicesResponsePayload } from '../../../../common/http_api/log_analysis';
|
import {
|
||||||
|
ValidationIndicesResponsePayload,
|
||||||
|
ValidateLogEntryDatasetsResponsePayload,
|
||||||
|
} from '../../../../common/http_api/log_analysis';
|
||||||
|
import { DatasetFilter } from '../../../../common/log_analysis';
|
||||||
|
|
||||||
export interface ModuleDescriptor<JobType extends string> {
|
export interface ModuleDescriptor<JobType extends string> {
|
||||||
moduleId: string;
|
moduleId: string;
|
||||||
|
@ -20,12 +24,20 @@ export interface ModuleDescriptor<JobType extends string> {
|
||||||
setUpModule: (
|
setUpModule: (
|
||||||
start: number | undefined,
|
start: number | undefined,
|
||||||
end: number | undefined,
|
end: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter,
|
||||||
sourceConfiguration: ModuleSourceConfiguration
|
sourceConfiguration: ModuleSourceConfiguration
|
||||||
) => Promise<SetupMlModuleResponsePayload>;
|
) => Promise<SetupMlModuleResponsePayload>;
|
||||||
cleanUpModule: (spaceId: string, sourceId: string) => Promise<DeleteJobsResponsePayload>;
|
cleanUpModule: (spaceId: string, sourceId: string) => Promise<DeleteJobsResponsePayload>;
|
||||||
validateSetupIndices: (
|
validateSetupIndices: (
|
||||||
sourceConfiguration: ModuleSourceConfiguration
|
indices: string[],
|
||||||
|
timestampField: string
|
||||||
) => Promise<ValidationIndicesResponsePayload>;
|
) => Promise<ValidationIndicesResponsePayload>;
|
||||||
|
validateSetupDatasets: (
|
||||||
|
indices: string[],
|
||||||
|
timestampField: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number
|
||||||
|
) => Promise<ValidateLogEntryDatasetsResponsePayload>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ModuleSourceConfiguration {
|
export interface ModuleSourceConfiguration {
|
||||||
|
|
|
@ -0,0 +1,264 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { isEqual } from 'lodash';
|
||||||
|
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||||
|
import { usePrevious } from 'react-use';
|
||||||
|
import {
|
||||||
|
combineDatasetFilters,
|
||||||
|
DatasetFilter,
|
||||||
|
filterDatasetFilter,
|
||||||
|
isExampleDataIndex,
|
||||||
|
} from '../../../../common/log_analysis';
|
||||||
|
import {
|
||||||
|
AvailableIndex,
|
||||||
|
ValidationIndicesError,
|
||||||
|
ValidationIndicesUIError,
|
||||||
|
} from '../../../components/logging/log_analysis_setup/initial_configuration_step';
|
||||||
|
import { useTrackedPromise } from '../../../utils/use_tracked_promise';
|
||||||
|
import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types';
|
||||||
|
|
||||||
|
type SetupHandler = (
|
||||||
|
indices: string[],
|
||||||
|
startTime: number | undefined,
|
||||||
|
endTime: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter
|
||||||
|
) => void;
|
||||||
|
|
||||||
|
interface AnalysisSetupStateArguments<JobType extends string> {
|
||||||
|
cleanUpAndSetUpModule: SetupHandler;
|
||||||
|
moduleDescriptor: ModuleDescriptor<JobType>;
|
||||||
|
setUpModule: SetupHandler;
|
||||||
|
sourceConfiguration: ModuleSourceConfiguration;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fourWeeksInMs = 86400000 * 7 * 4;
|
||||||
|
|
||||||
|
export const useAnalysisSetupState = <JobType extends string>({
|
||||||
|
cleanUpAndSetUpModule,
|
||||||
|
moduleDescriptor: { validateSetupDatasets, validateSetupIndices },
|
||||||
|
setUpModule,
|
||||||
|
sourceConfiguration,
|
||||||
|
}: AnalysisSetupStateArguments<JobType>) => {
|
||||||
|
const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
|
||||||
|
const [endTime, setEndTime] = useState<number | undefined>(undefined);
|
||||||
|
|
||||||
|
const [validatedIndices, setValidatedIndices] = useState<AvailableIndex[]>(
|
||||||
|
sourceConfiguration.indices.map(indexName => ({
|
||||||
|
name: indexName,
|
||||||
|
validity: 'unknown' as const,
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
|
||||||
|
const updateIndicesWithValidationErrors = useCallback(
|
||||||
|
(validationErrors: ValidationIndicesError[]) =>
|
||||||
|
setValidatedIndices(availableIndices =>
|
||||||
|
availableIndices.map(previousAvailableIndex => {
|
||||||
|
const indexValiationErrors = validationErrors.filter(
|
||||||
|
({ index }) => index === previousAvailableIndex.name
|
||||||
|
);
|
||||||
|
|
||||||
|
if (indexValiationErrors.length > 0) {
|
||||||
|
return {
|
||||||
|
validity: 'invalid',
|
||||||
|
name: previousAvailableIndex.name,
|
||||||
|
errors: indexValiationErrors,
|
||||||
|
};
|
||||||
|
} else if (previousAvailableIndex.validity === 'valid') {
|
||||||
|
return {
|
||||||
|
...previousAvailableIndex,
|
||||||
|
validity: 'valid',
|
||||||
|
errors: [],
|
||||||
|
};
|
||||||
|
} else {
|
||||||
|
return {
|
||||||
|
validity: 'valid',
|
||||||
|
name: previousAvailableIndex.name,
|
||||||
|
isSelected: !isExampleDataIndex(previousAvailableIndex.name),
|
||||||
|
availableDatasets: [],
|
||||||
|
datasetFilter: {
|
||||||
|
type: 'includeAll' as const,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
})
|
||||||
|
),
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
|
const updateIndicesWithAvailableDatasets = useCallback(
|
||||||
|
(availableDatasets: Array<{ indexName: string; datasets: string[] }>) =>
|
||||||
|
setValidatedIndices(availableIndices =>
|
||||||
|
availableIndices.map(previousAvailableIndex => {
|
||||||
|
if (previousAvailableIndex.validity !== 'valid') {
|
||||||
|
return previousAvailableIndex;
|
||||||
|
}
|
||||||
|
|
||||||
|
const availableDatasetsForIndex = availableDatasets.filter(
|
||||||
|
({ indexName }) => indexName === previousAvailableIndex.name
|
||||||
|
);
|
||||||
|
const newAvailableDatasets = availableDatasetsForIndex.flatMap(
|
||||||
|
({ datasets }) => datasets
|
||||||
|
);
|
||||||
|
|
||||||
|
// filter out datasets that have disappeared if this index' datasets were updated
|
||||||
|
const newDatasetFilter: DatasetFilter =
|
||||||
|
availableDatasetsForIndex.length > 0
|
||||||
|
? filterDatasetFilter(previousAvailableIndex.datasetFilter, dataset =>
|
||||||
|
newAvailableDatasets.includes(dataset)
|
||||||
|
)
|
||||||
|
: previousAvailableIndex.datasetFilter;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...previousAvailableIndex,
|
||||||
|
availableDatasets: newAvailableDatasets,
|
||||||
|
datasetFilter: newDatasetFilter,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
),
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
|
const validIndexNames = useMemo(
|
||||||
|
() => validatedIndices.filter(index => index.validity === 'valid').map(index => index.name),
|
||||||
|
[validatedIndices]
|
||||||
|
);
|
||||||
|
|
||||||
|
const selectedIndexNames = useMemo(
|
||||||
|
() =>
|
||||||
|
validatedIndices
|
||||||
|
.filter(index => index.validity === 'valid' && index.isSelected)
|
||||||
|
.map(i => i.name),
|
||||||
|
[validatedIndices]
|
||||||
|
);
|
||||||
|
|
||||||
|
const datasetFilter = useMemo(
|
||||||
|
() =>
|
||||||
|
validatedIndices
|
||||||
|
.flatMap(validatedIndex =>
|
||||||
|
validatedIndex.validity === 'valid'
|
||||||
|
? validatedIndex.datasetFilter
|
||||||
|
: { type: 'includeAll' as const }
|
||||||
|
)
|
||||||
|
.reduce(combineDatasetFilters, { type: 'includeAll' as const }),
|
||||||
|
[validatedIndices]
|
||||||
|
);
|
||||||
|
|
||||||
|
const [validateIndicesRequest, validateIndices] = useTrackedPromise(
|
||||||
|
{
|
||||||
|
cancelPreviousOn: 'resolution',
|
||||||
|
createPromise: async () => {
|
||||||
|
return await validateSetupIndices(
|
||||||
|
sourceConfiguration.indices,
|
||||||
|
sourceConfiguration.timestampField
|
||||||
|
);
|
||||||
|
},
|
||||||
|
onResolve: ({ data: { errors } }) => {
|
||||||
|
updateIndicesWithValidationErrors(errors);
|
||||||
|
},
|
||||||
|
onReject: () => {
|
||||||
|
setValidatedIndices([]);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[sourceConfiguration.indices, sourceConfiguration.timestampField]
|
||||||
|
);
|
||||||
|
|
||||||
|
const [validateDatasetsRequest, validateDatasets] = useTrackedPromise(
|
||||||
|
{
|
||||||
|
cancelPreviousOn: 'resolution',
|
||||||
|
createPromise: async () => {
|
||||||
|
if (validIndexNames.length === 0) {
|
||||||
|
return { data: { datasets: [] } };
|
||||||
|
}
|
||||||
|
|
||||||
|
return await validateSetupDatasets(
|
||||||
|
validIndexNames,
|
||||||
|
sourceConfiguration.timestampField,
|
||||||
|
startTime ?? 0,
|
||||||
|
endTime ?? Date.now()
|
||||||
|
);
|
||||||
|
},
|
||||||
|
onResolve: ({ data: { datasets } }) => {
|
||||||
|
updateIndicesWithAvailableDatasets(datasets);
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[validIndexNames, sourceConfiguration.timestampField, startTime, endTime]
|
||||||
|
);
|
||||||
|
|
||||||
|
const setUp = useCallback(() => {
|
||||||
|
return setUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
|
||||||
|
}, [setUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);
|
||||||
|
|
||||||
|
const cleanUpAndSetUp = useCallback(() => {
|
||||||
|
return cleanUpAndSetUpModule(selectedIndexNames, startTime, endTime, datasetFilter);
|
||||||
|
}, [cleanUpAndSetUpModule, selectedIndexNames, startTime, endTime, datasetFilter]);
|
||||||
|
|
||||||
|
const isValidating = useMemo(
|
||||||
|
() => validateIndicesRequest.state === 'pending' || validateDatasetsRequest.state === 'pending',
|
||||||
|
[validateDatasetsRequest.state, validateIndicesRequest.state]
|
||||||
|
);
|
||||||
|
|
||||||
|
const validationErrors = useMemo<ValidationIndicesUIError[]>(() => {
|
||||||
|
if (isValidating) {
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (validateIndicesRequest.state === 'rejected') {
|
||||||
|
return [{ error: 'NETWORK_ERROR' }];
|
||||||
|
}
|
||||||
|
|
||||||
|
if (selectedIndexNames.length === 0) {
|
||||||
|
return [{ error: 'TOO_FEW_SELECTED_INDICES' }];
|
||||||
|
}
|
||||||
|
|
||||||
|
return validatedIndices.reduce<ValidationIndicesUIError[]>((errors, index) => {
|
||||||
|
return index.validity === 'invalid' && selectedIndexNames.includes(index.name)
|
||||||
|
? [...errors, ...index.errors]
|
||||||
|
: errors;
|
||||||
|
}, []);
|
||||||
|
}, [isValidating, validateIndicesRequest.state, selectedIndexNames, validatedIndices]);
|
||||||
|
|
||||||
|
const prevStartTime = usePrevious(startTime);
|
||||||
|
const prevEndTime = usePrevious(endTime);
|
||||||
|
const prevValidIndexNames = usePrevious(validIndexNames);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
validateIndices();
|
||||||
|
}, [validateIndices]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (
|
||||||
|
startTime !== prevStartTime ||
|
||||||
|
endTime !== prevEndTime ||
|
||||||
|
!isEqual(validIndexNames, prevValidIndexNames)
|
||||||
|
) {
|
||||||
|
validateDatasets();
|
||||||
|
}
|
||||||
|
}, [
|
||||||
|
endTime,
|
||||||
|
prevEndTime,
|
||||||
|
prevStartTime,
|
||||||
|
prevValidIndexNames,
|
||||||
|
startTime,
|
||||||
|
validIndexNames,
|
||||||
|
validateDatasets,
|
||||||
|
]);
|
||||||
|
|
||||||
|
return {
|
||||||
|
cleanUpAndSetUp,
|
||||||
|
datasetFilter,
|
||||||
|
endTime,
|
||||||
|
isValidating,
|
||||||
|
selectedIndexNames,
|
||||||
|
setEndTime,
|
||||||
|
setStartTime,
|
||||||
|
setUp,
|
||||||
|
startTime,
|
||||||
|
validatedIndices,
|
||||||
|
setValidatedIndices,
|
||||||
|
validationErrors,
|
||||||
|
};
|
||||||
|
};
|
|
@ -1,142 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
||||||
* or more contributor license agreements. Licensed under the Elastic License;
|
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
|
||||||
|
|
||||||
import { isExampleDataIndex } from '../../../../common/log_analysis';
|
|
||||||
import {
|
|
||||||
ValidatedIndex,
|
|
||||||
ValidationIndicesUIError,
|
|
||||||
} from '../../../components/logging/log_analysis_setup/initial_configuration_step';
|
|
||||||
import { useTrackedPromise } from '../../../utils/use_tracked_promise';
|
|
||||||
import { ModuleDescriptor, ModuleSourceConfiguration } from './log_analysis_module_types';
|
|
||||||
|
|
||||||
type SetupHandler = (
|
|
||||||
indices: string[],
|
|
||||||
startTime: number | undefined,
|
|
||||||
endTime: number | undefined
|
|
||||||
) => void;
|
|
||||||
|
|
||||||
interface AnalysisSetupStateArguments<JobType extends string> {
|
|
||||||
cleanUpAndSetUpModule: SetupHandler;
|
|
||||||
moduleDescriptor: ModuleDescriptor<JobType>;
|
|
||||||
setUpModule: SetupHandler;
|
|
||||||
sourceConfiguration: ModuleSourceConfiguration;
|
|
||||||
}
|
|
||||||
|
|
||||||
const fourWeeksInMs = 86400000 * 7 * 4;
|
|
||||||
|
|
||||||
export const useAnalysisSetupState = <JobType extends string>({
|
|
||||||
cleanUpAndSetUpModule,
|
|
||||||
moduleDescriptor: { validateSetupIndices },
|
|
||||||
setUpModule,
|
|
||||||
sourceConfiguration,
|
|
||||||
}: AnalysisSetupStateArguments<JobType>) => {
|
|
||||||
const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
|
|
||||||
const [endTime, setEndTime] = useState<number | undefined>(undefined);
|
|
||||||
|
|
||||||
const [validatedIndices, setValidatedIndices] = useState<ValidatedIndex[]>([]);
|
|
||||||
|
|
||||||
const [validateIndicesRequest, validateIndices] = useTrackedPromise(
|
|
||||||
{
|
|
||||||
cancelPreviousOn: 'resolution',
|
|
||||||
createPromise: async () => {
|
|
||||||
return await validateSetupIndices(sourceConfiguration);
|
|
||||||
},
|
|
||||||
onResolve: ({ data: { errors } }) => {
|
|
||||||
setValidatedIndices(previousValidatedIndices =>
|
|
||||||
sourceConfiguration.indices.map(indexName => {
|
|
||||||
const previousValidatedIndex = previousValidatedIndices.filter(
|
|
||||||
({ name }) => name === indexName
|
|
||||||
)[0];
|
|
||||||
const indexValiationErrors = errors.filter(({ index }) => index === indexName);
|
|
||||||
if (indexValiationErrors.length > 0) {
|
|
||||||
return {
|
|
||||||
validity: 'invalid',
|
|
||||||
name: indexName,
|
|
||||||
errors: indexValiationErrors,
|
|
||||||
};
|
|
||||||
} else {
|
|
||||||
return {
|
|
||||||
validity: 'valid',
|
|
||||||
name: indexName,
|
|
||||||
isSelected:
|
|
||||||
previousValidatedIndex?.validity === 'valid'
|
|
||||||
? previousValidatedIndex?.isSelected
|
|
||||||
: !isExampleDataIndex(indexName),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
})
|
|
||||||
);
|
|
||||||
},
|
|
||||||
onReject: () => {
|
|
||||||
setValidatedIndices([]);
|
|
||||||
},
|
|
||||||
},
|
|
||||||
[sourceConfiguration.indices]
|
|
||||||
);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
validateIndices();
|
|
||||||
}, [validateIndices]);
|
|
||||||
|
|
||||||
const selectedIndexNames = useMemo(
|
|
||||||
() =>
|
|
||||||
validatedIndices
|
|
||||||
.filter(index => index.validity === 'valid' && index.isSelected)
|
|
||||||
.map(i => i.name),
|
|
||||||
[validatedIndices]
|
|
||||||
);
|
|
||||||
|
|
||||||
const setUp = useCallback(() => {
|
|
||||||
return setUpModule(selectedIndexNames, startTime, endTime);
|
|
||||||
}, [setUpModule, selectedIndexNames, startTime, endTime]);
|
|
||||||
|
|
||||||
const cleanUpAndSetUp = useCallback(() => {
|
|
||||||
return cleanUpAndSetUpModule(selectedIndexNames, startTime, endTime);
|
|
||||||
}, [cleanUpAndSetUpModule, selectedIndexNames, startTime, endTime]);
|
|
||||||
|
|
||||||
const isValidating = useMemo(
|
|
||||||
() =>
|
|
||||||
validateIndicesRequest.state === 'pending' ||
|
|
||||||
validateIndicesRequest.state === 'uninitialized',
|
|
||||||
[validateIndicesRequest.state]
|
|
||||||
);
|
|
||||||
|
|
||||||
const validationErrors = useMemo<ValidationIndicesUIError[]>(() => {
|
|
||||||
if (isValidating) {
|
|
||||||
return [];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (validateIndicesRequest.state === 'rejected') {
|
|
||||||
return [{ error: 'NETWORK_ERROR' }];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (selectedIndexNames.length === 0) {
|
|
||||||
return [{ error: 'TOO_FEW_SELECTED_INDICES' }];
|
|
||||||
}
|
|
||||||
|
|
||||||
return validatedIndices.reduce<ValidationIndicesUIError[]>((errors, index) => {
|
|
||||||
return index.validity === 'invalid' && selectedIndexNames.includes(index.name)
|
|
||||||
? [...errors, ...index.errors]
|
|
||||||
: errors;
|
|
||||||
}, []);
|
|
||||||
}, [isValidating, validateIndicesRequest.state, selectedIndexNames, validatedIndices]);
|
|
||||||
|
|
||||||
return {
|
|
||||||
cleanUpAndSetUp,
|
|
||||||
endTime,
|
|
||||||
isValidating,
|
|
||||||
selectedIndexNames,
|
|
||||||
setEndTime,
|
|
||||||
setStartTime,
|
|
||||||
setUp,
|
|
||||||
startTime,
|
|
||||||
validatedIndices,
|
|
||||||
setValidatedIndices,
|
|
||||||
validationErrors,
|
|
||||||
};
|
|
||||||
};
|
|
|
@ -7,20 +7,21 @@
|
||||||
import {
|
import {
|
||||||
bucketSpan,
|
bucketSpan,
|
||||||
categoriesMessageField,
|
categoriesMessageField,
|
||||||
|
DatasetFilter,
|
||||||
getJobId,
|
getJobId,
|
||||||
LogEntryCategoriesJobType,
|
LogEntryCategoriesJobType,
|
||||||
logEntryCategoriesJobTypes,
|
logEntryCategoriesJobTypes,
|
||||||
partitionField,
|
partitionField,
|
||||||
} from '../../../../common/log_analysis';
|
} from '../../../../common/log_analysis';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
cleanUpJobsAndDatafeeds,
|
||||||
ModuleDescriptor,
|
ModuleDescriptor,
|
||||||
ModuleSourceConfiguration,
|
ModuleSourceConfiguration,
|
||||||
cleanUpJobsAndDatafeeds,
|
|
||||||
} from '../../../containers/logs/log_analysis';
|
} from '../../../containers/logs/log_analysis';
|
||||||
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
|
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
|
||||||
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
|
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
|
||||||
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
|
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
|
||||||
|
import { callValidateDatasetsAPI } from '../../../containers/logs/log_analysis/api/validate_datasets';
|
||||||
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';
|
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';
|
||||||
|
|
||||||
const moduleId = 'logs_ui_categories';
|
const moduleId = 'logs_ui_categories';
|
||||||
|
@ -48,6 +49,7 @@ const getModuleDefinition = async () => {
|
||||||
const setUpModule = async (
|
const setUpModule = async (
|
||||||
start: number | undefined,
|
start: number | undefined,
|
||||||
end: number | undefined,
|
end: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter,
|
||||||
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration
|
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration
|
||||||
) => {
|
) => {
|
||||||
const indexNamePattern = indices.join(',');
|
const indexNamePattern = indices.join(',');
|
||||||
|
@ -65,10 +67,31 @@ const setUpModule = async (
|
||||||
indexPattern: indexNamePattern,
|
indexPattern: indexNamePattern,
|
||||||
timestampField,
|
timestampField,
|
||||||
bucketSpan,
|
bucketSpan,
|
||||||
|
datasetFilter,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
const query = {
|
||||||
|
bool: {
|
||||||
|
filter: [
|
||||||
|
...(datasetFilter.type === 'includeSome'
|
||||||
|
? [
|
||||||
|
{
|
||||||
|
terms: {
|
||||||
|
'event.dataset': datasetFilter.datasets,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
: []),
|
||||||
|
{
|
||||||
|
exists: {
|
||||||
|
field: 'message',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
return callSetupMlModuleAPI(
|
return callSetupMlModuleAPI(
|
||||||
moduleId,
|
moduleId,
|
||||||
|
@ -77,7 +100,9 @@ const setUpModule = async (
|
||||||
spaceId,
|
spaceId,
|
||||||
sourceId,
|
sourceId,
|
||||||
indexNamePattern,
|
indexNamePattern,
|
||||||
jobOverrides
|
jobOverrides,
|
||||||
|
[],
|
||||||
|
query
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -85,7 +110,7 @@ const cleanUpModule = async (spaceId: string, sourceId: string) => {
|
||||||
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryCategoriesJobTypes);
|
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryCategoriesJobTypes);
|
||||||
};
|
};
|
||||||
|
|
||||||
const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => {
|
const validateSetupIndices = async (indices: string[], timestampField: string) => {
|
||||||
return await callValidateIndicesAPI(indices, [
|
return await callValidateIndicesAPI(indices, [
|
||||||
{
|
{
|
||||||
name: timestampField,
|
name: timestampField,
|
||||||
|
@ -102,6 +127,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon
|
||||||
]);
|
]);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const validateSetupDatasets = async (
|
||||||
|
indices: string[],
|
||||||
|
timestampField: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number
|
||||||
|
) => {
|
||||||
|
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
|
||||||
|
};
|
||||||
|
|
||||||
export const logEntryCategoriesModule: ModuleDescriptor<LogEntryCategoriesJobType> = {
|
export const logEntryCategoriesModule: ModuleDescriptor<LogEntryCategoriesJobType> = {
|
||||||
moduleId,
|
moduleId,
|
||||||
jobTypes: logEntryCategoriesJobTypes,
|
jobTypes: logEntryCategoriesJobTypes,
|
||||||
|
@ -111,5 +145,6 @@ export const logEntryCategoriesModule: ModuleDescriptor<LogEntryCategoriesJobTyp
|
||||||
getModuleDefinition,
|
getModuleDefinition,
|
||||||
setUpModule,
|
setUpModule,
|
||||||
cleanUpModule,
|
cleanUpModule,
|
||||||
|
validateSetupDatasets,
|
||||||
validateSetupIndices,
|
validateSetupIndices,
|
||||||
};
|
};
|
||||||
|
|
|
@ -55,7 +55,7 @@ export const LogEntryCategoriesSetupContent: React.FunctionComponent = () => {
|
||||||
createProcessStep({
|
createProcessStep({
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
errorMessages: lastSetupErrorMessages,
|
errorMessages: lastSetupErrorMessages,
|
||||||
isConfigurationValid: validationErrors.length <= 0,
|
isConfigurationValid: validationErrors.length <= 0 && !isValidating,
|
||||||
setUp,
|
setUp,
|
||||||
setupStatus,
|
setupStatus,
|
||||||
viewResults,
|
viewResults,
|
||||||
|
|
|
@ -6,20 +6,21 @@
|
||||||
|
|
||||||
import {
|
import {
|
||||||
bucketSpan,
|
bucketSpan,
|
||||||
|
DatasetFilter,
|
||||||
getJobId,
|
getJobId,
|
||||||
LogEntryRateJobType,
|
LogEntryRateJobType,
|
||||||
logEntryRateJobTypes,
|
logEntryRateJobTypes,
|
||||||
partitionField,
|
partitionField,
|
||||||
} from '../../../../common/log_analysis';
|
} from '../../../../common/log_analysis';
|
||||||
|
|
||||||
import {
|
import {
|
||||||
|
cleanUpJobsAndDatafeeds,
|
||||||
ModuleDescriptor,
|
ModuleDescriptor,
|
||||||
ModuleSourceConfiguration,
|
ModuleSourceConfiguration,
|
||||||
cleanUpJobsAndDatafeeds,
|
|
||||||
} from '../../../containers/logs/log_analysis';
|
} from '../../../containers/logs/log_analysis';
|
||||||
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
|
import { callJobsSummaryAPI } from '../../../containers/logs/log_analysis/api/ml_get_jobs_summary_api';
|
||||||
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
|
import { callGetMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_get_module';
|
||||||
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
|
import { callSetupMlModuleAPI } from '../../../containers/logs/log_analysis/api/ml_setup_module_api';
|
||||||
|
import { callValidateDatasetsAPI } from '../../../containers/logs/log_analysis/api/validate_datasets';
|
||||||
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';
|
import { callValidateIndicesAPI } from '../../../containers/logs/log_analysis/api/validate_indices';
|
||||||
|
|
||||||
const moduleId = 'logs_ui_analysis';
|
const moduleId = 'logs_ui_analysis';
|
||||||
|
@ -47,6 +48,7 @@ const getModuleDefinition = async () => {
|
||||||
const setUpModule = async (
|
const setUpModule = async (
|
||||||
start: number | undefined,
|
start: number | undefined,
|
||||||
end: number | undefined,
|
end: number | undefined,
|
||||||
|
datasetFilter: DatasetFilter,
|
||||||
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration
|
{ spaceId, sourceId, indices, timestampField }: ModuleSourceConfiguration
|
||||||
) => {
|
) => {
|
||||||
const indexNamePattern = indices.join(',');
|
const indexNamePattern = indices.join(',');
|
||||||
|
@ -68,6 +70,20 @@ const setUpModule = async (
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
const query =
|
||||||
|
datasetFilter.type === 'includeSome'
|
||||||
|
? {
|
||||||
|
bool: {
|
||||||
|
filter: [
|
||||||
|
{
|
||||||
|
terms: {
|
||||||
|
'event.dataset': datasetFilter.datasets,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
|
||||||
return callSetupMlModuleAPI(
|
return callSetupMlModuleAPI(
|
||||||
moduleId,
|
moduleId,
|
||||||
|
@ -76,7 +92,9 @@ const setUpModule = async (
|
||||||
spaceId,
|
spaceId,
|
||||||
sourceId,
|
sourceId,
|
||||||
indexNamePattern,
|
indexNamePattern,
|
||||||
jobOverrides
|
jobOverrides,
|
||||||
|
[],
|
||||||
|
query
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -84,7 +102,7 @@ const cleanUpModule = async (spaceId: string, sourceId: string) => {
|
||||||
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryRateJobTypes);
|
return await cleanUpJobsAndDatafeeds(spaceId, sourceId, logEntryRateJobTypes);
|
||||||
};
|
};
|
||||||
|
|
||||||
const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceConfiguration) => {
|
const validateSetupIndices = async (indices: string[], timestampField: string) => {
|
||||||
return await callValidateIndicesAPI(indices, [
|
return await callValidateIndicesAPI(indices, [
|
||||||
{
|
{
|
||||||
name: timestampField,
|
name: timestampField,
|
||||||
|
@ -97,6 +115,15 @@ const validateSetupIndices = async ({ indices, timestampField }: ModuleSourceCon
|
||||||
]);
|
]);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const validateSetupDatasets = async (
|
||||||
|
indices: string[],
|
||||||
|
timestampField: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number
|
||||||
|
) => {
|
||||||
|
return await callValidateDatasetsAPI(indices, timestampField, startTime, endTime);
|
||||||
|
};
|
||||||
|
|
||||||
export const logEntryRateModule: ModuleDescriptor<LogEntryRateJobType> = {
|
export const logEntryRateModule: ModuleDescriptor<LogEntryRateJobType> = {
|
||||||
moduleId,
|
moduleId,
|
||||||
jobTypes: logEntryRateJobTypes,
|
jobTypes: logEntryRateJobTypes,
|
||||||
|
@ -106,5 +133,6 @@ export const logEntryRateModule: ModuleDescriptor<LogEntryRateJobType> = {
|
||||||
getModuleDefinition,
|
getModuleDefinition,
|
||||||
setUpModule,
|
setUpModule,
|
||||||
cleanUpModule,
|
cleanUpModule,
|
||||||
|
validateSetupDatasets,
|
||||||
validateSetupIndices,
|
validateSetupIndices,
|
||||||
};
|
};
|
||||||
|
|
|
@ -55,7 +55,7 @@ export const LogEntryRateSetupContent: React.FunctionComponent = () => {
|
||||||
createProcessStep({
|
createProcessStep({
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
errorMessages: lastSetupErrorMessages,
|
errorMessages: lastSetupErrorMessages,
|
||||||
isConfigurationValid: validationErrors.length <= 0,
|
isConfigurationValid: validationErrors.length <= 0 && !isValidating,
|
||||||
setUp,
|
setUp,
|
||||||
setupStatus,
|
setupStatus,
|
||||||
viewResults,
|
viewResults,
|
||||||
|
|
|
@ -15,6 +15,7 @@ import {
|
||||||
initGetLogEntryCategoryDatasetsRoute,
|
initGetLogEntryCategoryDatasetsRoute,
|
||||||
initGetLogEntryCategoryExamplesRoute,
|
initGetLogEntryCategoryExamplesRoute,
|
||||||
initGetLogEntryRateRoute,
|
initGetLogEntryRateRoute,
|
||||||
|
initValidateLogAnalysisDatasetsRoute,
|
||||||
initValidateLogAnalysisIndicesRoute,
|
initValidateLogAnalysisIndicesRoute,
|
||||||
} from './routes/log_analysis';
|
} from './routes/log_analysis';
|
||||||
import { initMetricExplorerRoute } from './routes/metrics_explorer';
|
import { initMetricExplorerRoute } from './routes/metrics_explorer';
|
||||||
|
@ -51,6 +52,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
|
||||||
initSnapshotRoute(libs);
|
initSnapshotRoute(libs);
|
||||||
initNodeDetailsRoute(libs);
|
initNodeDetailsRoute(libs);
|
||||||
initSourceRoute(libs);
|
initSourceRoute(libs);
|
||||||
|
initValidateLogAnalysisDatasetsRoute(libs);
|
||||||
initValidateLogAnalysisIndicesRoute(libs);
|
initValidateLogAnalysisIndicesRoute(libs);
|
||||||
initLogEntriesRoute(libs);
|
initLogEntriesRoute(libs);
|
||||||
initLogEntriesHighlightsRoute(libs);
|
initLogEntriesHighlightsRoute(libs);
|
||||||
|
|
|
@ -38,6 +38,7 @@ export function compose(core: CoreSetup, config: InfraConfig, plugins: InfraServ
|
||||||
sources,
|
sources,
|
||||||
}),
|
}),
|
||||||
logEntries: new InfraLogEntriesDomain(new InfraKibanaLogEntriesAdapter(framework), {
|
logEntries: new InfraLogEntriesDomain(new InfraKibanaLogEntriesAdapter(framework), {
|
||||||
|
framework,
|
||||||
sources,
|
sources,
|
||||||
}),
|
}),
|
||||||
metrics: new InfraMetricsDomain(new KibanaMetricsAdapter(framework)),
|
metrics: new InfraMetricsDomain(new KibanaMetricsAdapter(framework)),
|
||||||
|
|
|
@ -29,6 +29,14 @@ import {
|
||||||
Highlights,
|
Highlights,
|
||||||
compileFormattingRules,
|
compileFormattingRules,
|
||||||
} from './message';
|
} from './message';
|
||||||
|
import { KibanaFramework } from '../../adapters/framework/kibana_framework_adapter';
|
||||||
|
import { decodeOrThrow } from '../../../../common/runtime_types';
|
||||||
|
import {
|
||||||
|
logEntryDatasetsResponseRT,
|
||||||
|
LogEntryDatasetBucket,
|
||||||
|
CompositeDatasetKey,
|
||||||
|
createLogEntryDatasetsQuery,
|
||||||
|
} from './queries/log_entry_datasets';
|
||||||
|
|
||||||
export interface LogEntriesParams {
|
export interface LogEntriesParams {
|
||||||
startTimestamp: number;
|
startTimestamp: number;
|
||||||
|
@ -51,10 +59,15 @@ export const LOG_ENTRIES_PAGE_SIZE = 200;
|
||||||
|
|
||||||
const FIELDS_FROM_CONTEXT = ['log.file.path', 'host.name', 'container.id'] as const;
|
const FIELDS_FROM_CONTEXT = ['log.file.path', 'host.name', 'container.id'] as const;
|
||||||
|
|
||||||
|
const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
|
||||||
|
|
||||||
export class InfraLogEntriesDomain {
|
export class InfraLogEntriesDomain {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly adapter: LogEntriesAdapter,
|
private readonly adapter: LogEntriesAdapter,
|
||||||
private readonly libs: { sources: InfraSources }
|
private readonly libs: {
|
||||||
|
framework: KibanaFramework;
|
||||||
|
sources: InfraSources;
|
||||||
|
}
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
public async getLogEntriesAround(
|
public async getLogEntriesAround(
|
||||||
|
@ -256,6 +269,45 @@ export class InfraLogEntriesDomain {
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public async getLogEntryDatasets(
|
||||||
|
requestContext: RequestHandlerContext,
|
||||||
|
timestampField: string,
|
||||||
|
indexName: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number
|
||||||
|
) {
|
||||||
|
let datasetBuckets: LogEntryDatasetBucket[] = [];
|
||||||
|
let afterLatestBatchKey: CompositeDatasetKey | undefined;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const datasetsReponse = await this.libs.framework.callWithRequest(
|
||||||
|
requestContext,
|
||||||
|
'search',
|
||||||
|
createLogEntryDatasetsQuery(
|
||||||
|
indexName,
|
||||||
|
timestampField,
|
||||||
|
startTime,
|
||||||
|
endTime,
|
||||||
|
COMPOSITE_AGGREGATION_BATCH_SIZE,
|
||||||
|
afterLatestBatchKey
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const { after_key: afterKey, buckets: latestBatchBuckets } = decodeOrThrow(
|
||||||
|
logEntryDatasetsResponseRT
|
||||||
|
)(datasetsReponse).aggregations.dataset_buckets;
|
||||||
|
|
||||||
|
datasetBuckets = [...datasetBuckets, ...latestBatchBuckets];
|
||||||
|
afterLatestBatchKey = afterKey;
|
||||||
|
|
||||||
|
if (latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return datasetBuckets.map(({ key: { dataset } }) => dataset);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
interface LogItemHit {
|
interface LogItemHit {
|
||||||
|
|
|
@ -0,0 +1,98 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as rt from 'io-ts';
|
||||||
|
|
||||||
|
import { commonSearchSuccessResponseFieldsRT } from '../../../../utils/elasticsearch_runtime_types';
|
||||||
|
|
||||||
|
export const createLogEntryDatasetsQuery = (
|
||||||
|
indexName: string,
|
||||||
|
timestampField: string,
|
||||||
|
startTime: number,
|
||||||
|
endTime: number,
|
||||||
|
size: number,
|
||||||
|
afterKey?: CompositeDatasetKey
|
||||||
|
) => ({
|
||||||
|
...defaultRequestParameters,
|
||||||
|
body: {
|
||||||
|
query: {
|
||||||
|
bool: {
|
||||||
|
filter: [
|
||||||
|
{
|
||||||
|
range: {
|
||||||
|
[timestampField]: {
|
||||||
|
gte: startTime,
|
||||||
|
lte: endTime,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
exists: {
|
||||||
|
field: 'event.dataset',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
aggs: {
|
||||||
|
dataset_buckets: {
|
||||||
|
composite: {
|
||||||
|
after: afterKey,
|
||||||
|
size,
|
||||||
|
sources: [
|
||||||
|
{
|
||||||
|
dataset: {
|
||||||
|
terms: {
|
||||||
|
field: 'event.dataset',
|
||||||
|
order: 'asc',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
index: indexName,
|
||||||
|
size: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
const defaultRequestParameters = {
|
||||||
|
allowNoIndices: true,
|
||||||
|
ignoreUnavailable: true,
|
||||||
|
trackScores: false,
|
||||||
|
trackTotalHits: false,
|
||||||
|
};
|
||||||
|
|
||||||
|
const compositeDatasetKeyRT = rt.type({
|
||||||
|
dataset: rt.string,
|
||||||
|
});
|
||||||
|
|
||||||
|
export type CompositeDatasetKey = rt.TypeOf<typeof compositeDatasetKeyRT>;
|
||||||
|
|
||||||
|
const logEntryDatasetBucketRT = rt.type({
|
||||||
|
key: compositeDatasetKeyRT,
|
||||||
|
});
|
||||||
|
|
||||||
|
export type LogEntryDatasetBucket = rt.TypeOf<typeof logEntryDatasetBucketRT>;
|
||||||
|
|
||||||
|
export const logEntryDatasetsResponseRT = rt.intersection([
|
||||||
|
commonSearchSuccessResponseFieldsRT,
|
||||||
|
rt.type({
|
||||||
|
aggregations: rt.type({
|
||||||
|
dataset_buckets: rt.intersection([
|
||||||
|
rt.type({
|
||||||
|
buckets: rt.array(logEntryDatasetBucketRT),
|
||||||
|
}),
|
||||||
|
rt.partial({
|
||||||
|
after_key: compositeDatasetKeyRT,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
export type LogEntryDatasetsResponse = rt.TypeOf<typeof logEntryDatasetsResponseRT>;
|
|
@ -119,6 +119,7 @@ export class InfraServerPlugin {
|
||||||
sources,
|
sources,
|
||||||
}),
|
}),
|
||||||
logEntries: new InfraLogEntriesDomain(new InfraKibanaLogEntriesAdapter(framework), {
|
logEntries: new InfraLogEntriesDomain(new InfraKibanaLogEntriesAdapter(framework), {
|
||||||
|
framework,
|
||||||
sources,
|
sources,
|
||||||
}),
|
}),
|
||||||
metrics: new InfraMetricsDomain(new KibanaMetricsAdapter(framework)),
|
metrics: new InfraMetricsDomain(new KibanaMetricsAdapter(framework)),
|
||||||
|
|
|
@ -0,0 +1,69 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
import Boom from 'boom';
|
||||||
|
|
||||||
|
import { InfraBackendLibs } from '../../../lib/infra_types';
|
||||||
|
import {
|
||||||
|
LOG_ANALYSIS_VALIDATE_DATASETS_PATH,
|
||||||
|
validateLogEntryDatasetsRequestPayloadRT,
|
||||||
|
validateLogEntryDatasetsResponsePayloadRT,
|
||||||
|
} from '../../../../common/http_api';
|
||||||
|
|
||||||
|
import { createValidationFunction } from '../../../../common/runtime_types';
|
||||||
|
|
||||||
|
export const initValidateLogAnalysisDatasetsRoute = ({
|
||||||
|
framework,
|
||||||
|
logEntries,
|
||||||
|
}: InfraBackendLibs) => {
|
||||||
|
framework.registerRoute(
|
||||||
|
{
|
||||||
|
method: 'post',
|
||||||
|
path: LOG_ANALYSIS_VALIDATE_DATASETS_PATH,
|
||||||
|
validate: {
|
||||||
|
body: createValidationFunction(validateLogEntryDatasetsRequestPayloadRT),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
framework.router.handleLegacyErrors(async (requestContext, request, response) => {
|
||||||
|
try {
|
||||||
|
const {
|
||||||
|
data: { indices, timestampField, startTime, endTime },
|
||||||
|
} = request.body;
|
||||||
|
|
||||||
|
const datasets = await Promise.all(
|
||||||
|
indices.map(async indexName => {
|
||||||
|
const indexDatasets = await logEntries.getLogEntryDatasets(
|
||||||
|
requestContext,
|
||||||
|
timestampField,
|
||||||
|
indexName,
|
||||||
|
startTime,
|
||||||
|
endTime
|
||||||
|
);
|
||||||
|
|
||||||
|
return {
|
||||||
|
indexName,
|
||||||
|
datasets: indexDatasets,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
|
return response.ok({
|
||||||
|
body: validateLogEntryDatasetsResponsePayloadRT.encode({ data: { datasets } }),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (Boom.isBoom(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.customError({
|
||||||
|
statusCode: error.statusCode ?? 500,
|
||||||
|
body: {
|
||||||
|
message: error.message ?? 'An unexpected error occurred',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
};
|
|
@ -4,4 +4,5 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
export * from './datasets';
|
||||||
export * from './indices';
|
export * from './indices';
|
||||||
|
|
|
@ -1,15 +1,4 @@
|
||||||
{
|
{
|
||||||
"job_id": "JOB_ID",
|
"job_id": "JOB_ID",
|
||||||
"indices": ["INDEX_PATTERN_NAME"],
|
"indices": ["INDEX_PATTERN_NAME"]
|
||||||
"query": {
|
|
||||||
"bool": {
|
|
||||||
"filter": [
|
|
||||||
{
|
|
||||||
"exists": {
|
|
||||||
"field": "message"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue