maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Logs UI] Add dataset-specific categorization warnings (#75351)

Browse source 
This adds dataset-specific categorization warnings for the categorization module. The warnings are displayed in call-outs on the relevant tabs as well as the job setup screens if a prior job with warnings exists. To that end this also changes the categorization job configuration to enable the partitioned categorization mode.

Co-authored-by: Alejandro Fernández Gómez <antarticonorte@gmail.com>
This commit is contained in:
Felix Stürmer 2020-09-24 17:02:23 +02:00 committed by GitHub
parent 5ff0c00529
commit 3f2e9f7705
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 1027 additions and 273 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/dev/storybook/aliases.ts
View file

@ -23,7 +23,7 @@ export const storybookAliases = {
codeeditor: 'src/plugins/kibana_react/public/code_editor/scripts/storybook.ts',
dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/scripts/storybook.js',
embeddable: 'src/plugins/embeddable/scripts/storybook.js',
infra: 'x-pack/legacy/plugins/infra/scripts/storybook.js',
infra: 'x-pack/plugins/infra/scripts/storybook.js',
security_solution: 'x-pack/plugins/security_solution/scripts/storybook.js',
ui_actions_enhanced: 'x-pack/plugins/ui_actions_enhanced/scripts/storybook.js',
observability: 'x-pack/plugins/observability/scripts/storybook.js',

1
x-pack/plugins/infra/common/http_api/log_analysis/results/index.ts
View file

@ -6,6 +6,7 @@
export * from './log_entry_categories';
export * from './log_entry_category_datasets';
export * from './log_entry_category_datasets_stats';
export * from './log_entry_category_examples';
export * from './log_entry_rate';
export * from './log_entry_examples';

72
x-pack/plugins/infra/common/http_api/log_analysis/results/log_entry_category_datasets_stats.ts Normal file
View file

@ -0,0 +1,72 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import * as rt from 'io-ts';
import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
export const LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH =
'/api/infra/log_analysis/results/latest_log_entry_category_datasets_stats';
const categorizerStatusRT = rt.keyof({
ok: null,
warn: null,
});
export type CategorizerStatus = rt.TypeOf<typeof categorizerStatusRT>;
/**
* request
*/
export const getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT = rt.type({
data: rt.type({
// the ids of the categorization jobs
jobIds: rt.array(rt.string),
// the time range to fetch the category datasets stats for
timeRange: timeRangeRT,
// the categorizer statuses to include stats for, empty means all
includeCategorizerStatuses: rt.array(categorizerStatusRT),
}),
});
export type GetLatestLogEntryCategoryDatasetsStatsRequestPayload = rt.TypeOf<
typeof getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT
>;
/**
* response
*/
const logEntryCategoriesDatasetStatsRT = rt.type({
categorization_status: categorizerStatusRT,
categorized_doc_count: rt.number,
dataset: rt.string,
dead_category_count: rt.number,
failed_category_count: rt.number,
frequent_category_count: rt.number,
job_id: rt.string,
log_time: rt.number,
rare_category_count: rt.number,
total_category_count: rt.number,
});
export type LogEntryCategoriesDatasetStats = rt.TypeOf<typeof logEntryCategoriesDatasetStatsRT>;
export const getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT = rt.intersection([
rt.type({
data: rt.type({
datasetStats: rt.array(logEntryCategoriesDatasetStatsRT),
}),
}),
rt.partial({
timing: routeTimingMetadataRT,
}),
]);
export type GetLatestLogEntryCategoryDatasetsStatsSuccessResponsePayload = rt.TypeOf<
typeof getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT
>;

1
x-pack/plugins/infra/common/log_analysis/index.ts
View file

@ -5,6 +5,7 @@
*/
export * from './log_analysis';
export * from './log_analysis_quality';
export * from './log_analysis_results';
export * from './log_entry_rate_analysis';
export * from './log_entry_categories_analysis';

42
x-pack/plugins/infra/common/log_analysis/log_analysis_quality.ts Normal file
View file

@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
interface ManyCategoriesWarningReason {
type: 'manyCategories';
categoriesDocumentRatio: number;
}
interface ManyDeadCategoriesWarningReason {
type: 'manyDeadCategories';
deadCategoriesRatio: number;
}
interface ManyRareCategoriesWarningReason {
type: 'manyRareCategories';
rareCategoriesRatio: number;
}
interface NoFrequentCategoriesWarningReason {
type: 'noFrequentCategories';
}
interface SingleCategoryWarningReason {
type: 'singleCategory';
}
export type CategoryQualityWarningReason =
| ManyCategoriesWarningReason
| ManyDeadCategoriesWarningReason
| ManyRareCategoriesWarningReason
| NoFrequentCategoriesWarningReason
| SingleCategoryWarningReason;
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
export interface CategoryQualityWarning {
type: 'categoryQualityWarning';
jobId: string;
dataset: string;
reasons: CategoryQualityWarningReason[];
}
export type QualityWarning = CategoryQualityWarning;

1
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx
View file

@ -31,6 +31,7 @@ export const JobConfigurationOutdatedCallout: React.FC<{
values={{
moduleName,
}}
tagName="p"
/>
</RecreateJobCallout>
);

1
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx
View file

@ -31,6 +31,7 @@ export const JobDefinitionOutdatedCallout: React.FC<{
values={{
moduleName,
}}
tagName="p"
/>
</RecreateJobCallout>
);

8
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/notices_section.tsx
View file

@ -5,7 +5,7 @@
*/
import React from 'react';
import { QualityWarning } from '../../../containers/logs/log_analysis/log_analysis_module_types';
import { QualityWarning } from '../../../../common/log_analysis';
import { LogAnalysisJobProblemIndicator } from './log_analysis_job_problem_indicator';
import { CategoryQualityWarnings } from './quality_warning_notices';
@ -41,6 +41,10 @@ export const CategoryJobNoticesSection: React.FC<{
onRecreateMlJobForReconfiguration={onRecreateMlJobForReconfiguration}
onRecreateMlJobForUpdate={onRecreateMlJobForUpdate}
/>
<CategoryQualityWarnings qualityWarnings={qualityWarnings} />
<CategoryQualityWarnings
hasSetupCapabilities={hasSetupCapabilities}
qualityWarnings={qualityWarnings}
onRecreateMlJob={onRecreateMlJobForReconfiguration}
/>
</>
);

68
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.stories.tsx Normal file
View file

@ -0,0 +1,68 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { action } from '@storybook/addon-actions';
import { storiesOf } from '@storybook/react';
import React from 'react';
import { EuiThemeProvider } from '../../../../../observability/public';
import { QualityWarning } from '../../../../common/log_analysis';
import { CategoryQualityWarnings } from './quality_warning_notices';
storiesOf('infra/logAnalysis/CategoryQualityWarnings', module)
.addDecorator((renderStory) => <EuiThemeProvider>{renderStory()}</EuiThemeProvider>)
.add('Partitioned warnings', () => {
return (
<CategoryQualityWarnings
hasSetupCapabilities={true}
onRecreateMlJob={action('on-recreate-ml-job')}
qualityWarnings={partitionedQualityWarnings}
/>
);
})
.add('Unpartitioned warnings', () => {
return (
<CategoryQualityWarnings
hasSetupCapabilities={true}
onRecreateMlJob={action('on-recreate-ml-job')}
qualityWarnings={unpartitionedQualityWarnings}
/>
);
});
const partitionedQualityWarnings: QualityWarning[] = [
{
type: 'categoryQualityWarning',
jobId: 'theMlJobId',
dataset: 'first.dataset',
reasons: [
{ type: 'singleCategory' },
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
],
},
{
type: 'categoryQualityWarning',
jobId: 'theMlJobId',
dataset: 'second.dataset',
reasons: [
{ type: 'noFrequentCategories' },
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.7 },
],
},
];
const unpartitionedQualityWarnings: QualityWarning[] = [
{
type: 'categoryQualityWarning',
jobId: 'theMlJobId',
dataset: '',
reasons: [
{ type: 'singleCategory' },
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
],
},
];

110
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/quality_warning_notices.tsx
View file

@ -4,43 +4,89 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { EuiCallOut } from '@elastic/eui';
import {
EuiAccordion,
EuiDescriptionList,
EuiDescriptionListDescription,
EuiDescriptionListTitle,
EuiSpacer,
htmlIdGenerator,
} from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import React from 'react';
import type {
import groupBy from 'lodash/groupBy';
import React, { Fragment, useState } from 'react';
import { euiStyled } from '../../../../../observability/public';
import {
CategoryQualityWarning,
CategoryQualityWarningReason,
QualityWarning,
} from '../../../containers/logs/log_analysis/log_analysis_module_types';
getFriendlyNameForPartitionId,
} from '../../../../common/log_analysis';
import { RecreateJobCallout } from './recreate_job_callout';
export const CategoryQualityWarnings: React.FC<{ qualityWarnings: QualityWarning[] }> = ({
qualityWarnings,
}) => (
<>
{qualityWarnings.map((qualityWarning, qualityWarningIndex) => (
<EuiCallOut
key={`${qualityWarningIndex}`}
title={categoryQualityWarningCalloutTitle}
color="warning"
iconType="alert"
>
<p>
export const CategoryQualityWarnings: React.FC<{
hasSetupCapabilities: boolean;
onRecreateMlJob: () => void;
qualityWarnings: CategoryQualityWarning[];
}> = ({ hasSetupCapabilities, onRecreateMlJob, qualityWarnings }) => {
const [detailAccordionId] = useState(htmlIdGenerator()());
const categoryQualityWarningsByJob = groupBy(qualityWarnings, 'jobId');
return (
<>
{Object.entries(categoryQualityWarningsByJob).map(([jobId, qualityWarningsForJob]) => (
<RecreateJobCallout
hasSetupCapabilities={hasSetupCapabilities}
key={`quality-warnings-callout-${jobId}`}
onRecreateMlJob={onRecreateMlJob}
title={categoryQualityWarningCalloutTitle}
>
<FormattedMessage
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningCalloutMessage"
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results."
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results. Consider excluding the respective datasets from the analysis."
tagName="p"
/>
</p>
<ul>
{qualityWarning.reasons.map((reason, reasonIndex) => (
<li key={`${reasonIndex}`}>
<CategoryQualityWarningReasonDescription reason={reason} />
</li>
))}
</ul>
</EuiCallOut>
))}
</>
);
<EuiAccordion
id={detailAccordionId}
buttonContent={
<FormattedMessage
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningDetailsAccordionButtonLabel"
defaultMessage="Details"
/>
}
paddingSize="m"
>
<EuiDescriptionList>
{qualityWarningsForJob.flatMap((qualityWarning) => (
<Fragment key={`item-${getFriendlyNameForPartitionId(qualityWarning.dataset)}`}>
<EuiDescriptionListTitle data-test-subj={`title-${qualityWarning.dataset}`}>
{getFriendlyNameForPartitionId(qualityWarning.dataset)}
</EuiDescriptionListTitle>
{qualityWarning.reasons.map((reason) => (
<QualityWarningReasonDescription
key={`description-${reason.type}-${qualityWarning.dataset}`}
data-test-subj={`description-${reason.type}-${qualityWarning.dataset}`}
>
<CategoryQualityWarningReasonDescription reason={reason} />
</QualityWarningReasonDescription>
))}
</Fragment>
))}
</EuiDescriptionList>
</EuiAccordion>
<EuiSpacer size="l" />
</RecreateJobCallout>
))}
</>
);
};
const QualityWarningReasonDescription = euiStyled(EuiDescriptionListDescription)`
display: list-item;
list-style-type: disc;
margin-left: ${(props) => props.theme.eui.paddingSizes.m};
`;
const categoryQualityWarningCalloutTitle = i18n.translate(
'xpack.infra.logs.logEntryCategories.categoryQUalityWarningCalloutTitle',
@ -49,7 +95,7 @@ const categoryQualityWarningCalloutTitle = i18n.translate(
}
);
const CategoryQualityWarningReasonDescription: React.FC<{
export const CategoryQualityWarningReasonDescription: React.FC<{
reason: CategoryQualityWarningReason;
}> = ({ reason }) => {
switch (reason.type) {
@ -57,7 +103,7 @@ const CategoryQualityWarningReasonDescription: React.FC<{
return (
<FormattedMessage
id="xpack.infra.logs.logEntryCategories.singleCategoryWarningReasonDescription"
defaultMessage="The analysis couldn't extract more than a single category from the log message."
defaultMessage="The analysis couldn't extract more than a single category from the log messages."
/>
);
case 'manyRareCategories':

2
x-pack/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx
View file

@ -14,7 +14,7 @@ export const RecreateJobCallout: React.FC<{
title?: React.ReactNode;
}> = ({ children, hasSetupCapabilities, onRecreateMlJob, title }) => (
<EuiCallOut color="warning" iconType="alert" title={title}>
<p>{children}</p>
{children}
<RecreateJobButton
color="warning"
hasSetupCapabilities={hasSetupCapabilities}

4
x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/analysis_setup_indices_form.tsx
View file

@ -8,6 +8,7 @@ import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { useCallback } from 'react';
import { QualityWarning } from '../../../../../common/log_analysis';
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
import { IndexSetupRow } from './index_setup_row';
import { AvailableIndex, ValidationIndicesError } from './validation';
@ -17,12 +18,14 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
indices: AvailableIndex[];
isValidating: boolean;
onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
previousQualityWarnings?: QualityWarning[];
validationErrors?: ValidationIndicesError[];
}> = ({
disabled = false,
indices,
isValidating,
onChangeSelectedIndices,
previousQualityWarnings = [],
validationErrors = [],
}) => {
const changeIsIndexSelected = useCallback(
@ -81,6 +84,7 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
key={index.name}
onChangeIsSelected={changeIsIndexSelected}
onChangeDatasetFilter={changeDatasetFilter}
previousQualityWarnings={previousQualityWarnings}
/>
))}
</>

30
x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_dataset_filter.tsx
View file

@ -7,6 +7,7 @@
import {
EuiFilterButton,
EuiFilterGroup,
EuiIconTip,
EuiPopover,
EuiPopoverTitle,
EuiSelectable,
@ -14,11 +15,15 @@ import {
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { useCallback, useMemo } from 'react';
import { DatasetFilter } from '../../../../../common/log_analysis';
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
import { useVisibilityState } from '../../../../utils/use_visibility_state';
import { CategoryQualityWarningReasonDescription } from '../../log_analysis_job_status/quality_warning_notices';
export const IndexSetupDatasetFilter: React.FC<{
availableDatasets: string[];
availableDatasets: Array<{
dataset: string;
warnings: QualityWarning[];
}>;
datasetFilter: DatasetFilter;
isDisabled?: boolean;
onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
@ -40,12 +45,13 @@ export const IndexSetupDatasetFilter: React.FC<{
[onChangeDatasetFilter]
);
const selectableOptions: EuiSelectableOption[] = useMemo(
const selectableOptions = useMemo<EuiSelectableOption[]>(
() =>
availableDatasets.map((datasetName) => ({
label: datasetName,
availableDatasets.map(({ dataset, warnings }) => ({
label: dataset,
append: warnings.length > 0 ? <DatasetWarningMarker warnings={warnings} /> : null,
checked:
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName)
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(dataset)
? 'on'
: undefined,
})),
@ -86,3 +92,15 @@ export const IndexSetupDatasetFilter: React.FC<{
</EuiFilterGroup>
);
};
const DatasetWarningMarker: React.FC<{ warnings: QualityWarning[] }> = ({ warnings }) => {
const warningDescriptions = warnings.flatMap((warning) =>
warning.type === 'categoryQualityWarning'
? warning.reasons.map((reason) => (
<CategoryQualityWarningReasonDescription key={reason.type} reason={reason} />
))
: []
);
return <EuiIconTip content={warningDescriptions} type="alert" color="warning" />;
};

62
x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/index_setup_row.tsx
View file

@ -4,10 +4,10 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiToolTip } from '@elastic/eui';
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIconTip } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { useCallback } from 'react';
import { DatasetFilter } from '../../../../../common/log_analysis';
import React, { useCallback, useMemo } from 'react';
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
import { IndexSetupDatasetFilter } from './index_setup_dataset_filter';
import { AvailableIndex, ValidationUIError } from './validation';
@ -16,7 +16,14 @@ export const IndexSetupRow: React.FC<{
isDisabled: boolean;
onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void;
onChangeIsSelected: (indexName: string, isSelected: boolean) => void;
}> = ({ index, isDisabled, onChangeDatasetFilter, onChangeIsSelected }) => {
previousQualityWarnings: QualityWarning[];
}> = ({
index,
isDisabled,
onChangeDatasetFilter,
onChangeIsSelected,
previousQualityWarnings,
}) => {
const changeIsSelected = useCallback(
(event: React.ChangeEvent<HTMLInputElement>) => {
onChangeIsSelected(index.name, event.currentTarget.checked);
@ -29,6 +36,29 @@ export const IndexSetupRow: React.FC<{
[index.name, onChangeDatasetFilter]
);
const datasets = useMemo(
() =>
index.validity === 'valid'
? index.availableDatasets.map((availableDataset) => ({
dataset: availableDataset,
warnings: previousQualityWarnings.filter(({ dataset }) => dataset === availableDataset),
}))
: [],
[index, previousQualityWarnings]
);
const datasetIndependentQualityWarnings = useMemo(
() => previousQualityWarnings.filter(({ dataset }) => dataset === ''),
[previousQualityWarnings]
);
const hasWarnings = useMemo(
() =>
datasetIndependentQualityWarnings.length > 0 ||
datasets.some(({ warnings }) => warnings.length > 0),
[datasetIndependentQualityWarnings, datasets]
);
const isSelected = index.validity === 'valid' && index.isSelected;
return (
@ -37,7 +67,23 @@ export const IndexSetupRow: React.FC<{
<EuiCheckbox
key={index.name}
id={index.name}
label={<EuiCode>{index.name}</EuiCode>}
label={
<>
<EuiCode>{index.name}</EuiCode>{' '}
{index.validity === 'valid' && hasWarnings ? (
<EuiIconTip
content={
<FormattedMessage
id="xpack.infra.logs.analsysisSetup.indexQualityWarningTooltipMessage"
defaultMessage="While analyzing the log messages from these indices we've detected some problems which might indicate a reduced quality of the results. Consider excluding these indices or problematic datasets from the analysis."
/>
}
type="alert"
color="warning"
/>
) : null}
</>
}
onChange={changeIsSelected}
checked={isSelected}
disabled={isDisabled || index.validity === 'invalid'}
@ -45,12 +91,10 @@ export const IndexSetupRow: React.FC<{
</EuiFlexItem>
<EuiFlexItem grow={false}>
{index.validity === 'invalid' ? (
<EuiToolTip content={formatValidationError(index.errors)}>
<EuiIcon type="alert" color="danger" />
</EuiToolTip>
<EuiIconTip content={formatValidationError(index.errors)} type="alert" color="danger" />
) : index.validity === 'valid' ? (
<IndexSetupDatasetFilter
availableDatasets={index.availableDatasets}
availableDatasets={datasets}
datasetFilter={index.datasetFilter}
isDisabled={!isSelected || isDisabled}
onChangeDatasetFilter={changeDatasetFilter}

104
x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.stories.tsx Normal file
View file

@ -0,0 +1,104 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { actions } from '@storybook/addon-actions';
import { storiesOf } from '@storybook/react';
import React from 'react';
import { EuiThemeProvider } from '../../../../../../observability/public';
import { InitialConfigurationStep } from './initial_configuration_step';
storiesOf('infra/logAnalysis/SetupInitialConfigurationStep', module)
.addDecorator((renderStory) => (
<EuiThemeProvider>
<div style={{ maxWidth: 800 }}>{renderStory()}</div>
</EuiThemeProvider>
))
.add('Reconfiguration with partitioned warnings', () => {
return (
<InitialConfigurationStep
{...storyActions}
startTime={Date.now()}
endTime={undefined}
isValidating={false}
setupStatus={{ type: 'required' }}
validatedIndices={[
{
name: 'index-1-*',
validity: 'valid',
isSelected: true,
datasetFilter: { type: 'includeAll' },
availableDatasets: ['first', 'second', 'third'],
},
{
name: 'index-2-*',
validity: 'invalid',
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
},
]}
previousQualityWarnings={[
{
type: 'categoryQualityWarning',
jobId: 'job-1',
dataset: 'second',
reasons: [
{ type: 'noFrequentCategories' },
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
],
},
{
type: 'categoryQualityWarning',
jobId: 'job-1',
dataset: 'third',
reasons: [{ type: 'singleCategory' }],
},
]}
/>
);
})
.add('Reconfiguration with unpartitioned warnings', () => {
return (
<InitialConfigurationStep
{...storyActions}
startTime={Date.now()}
endTime={undefined}
isValidating={false}
setupStatus={{ type: 'required' }}
validatedIndices={[
{
name: 'index-1-*',
validity: 'valid',
isSelected: true,
datasetFilter: { type: 'includeAll' },
availableDatasets: ['first', 'second', 'third'],
},
{
name: 'index-2-*',
validity: 'invalid',
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
},
]}
previousQualityWarnings={[
{
type: 'categoryQualityWarning',
jobId: 'job-1',
dataset: '',
reasons: [
{ type: 'noFrequentCategories' },
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
],
},
{
type: 'categoryQualityWarning',
jobId: 'job-1',
dataset: '',
reasons: [{ type: 'singleCategory' }],
},
]}
/>
);
});
const storyActions = actions('setStartTime', 'setEndTime', 'setValidatedIndices');

5
x-pack/plugins/infra/public/components/logging/log_analysis_setup/initial_configuration_step/initial_configuration_step.tsx
View file

@ -9,7 +9,7 @@ import { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps';
import { i18n } from '@kbn/i18n';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { useMemo } from 'react';
import { SetupStatus } from '../../../../../common/log_analysis';
import { QualityWarning, SetupStatus } from '../../../../../common/log_analysis';
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
import {
@ -31,6 +31,7 @@ interface InitialConfigurationStepProps {
setupStatus: SetupStatus;
setValidatedIndices: (selectedIndices: AvailableIndex[]) => void;
validationErrors?: ValidationUIError[];
previousQualityWarnings?: QualityWarning[];
}
export const createInitialConfigurationStep = (
@ -50,6 +51,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
setupStatus,
setValidatedIndices,
validationErrors = [],
previousQualityWarnings = [],
}: InitialConfigurationStepProps) => {
const disabled = useMemo(() => !editableFormStatus.includes(setupStatus.type), [setupStatus]);
@ -75,6 +77,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
indices={validatedIndices}
isValidating={isValidating}
onChangeSelectedIndices={setValidatedIndices}
previousQualityWarnings={previousQualityWarnings}
validationErrors={indexValidationErrors}
/>

9
x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/log_entry_categories_setup_view.tsx
View file

@ -6,6 +6,7 @@
import { EuiSpacer, EuiSteps, EuiText, EuiTitle } from '@elastic/eui';
import React, { useCallback, useMemo } from 'react';
import { useMount } from 'react-use';
import { useLogEntryCategoriesSetup } from '../../../../containers/logs/log_analysis/modules/log_entry_categories';
import { createInitialConfigurationStep } from '../initial_configuration_step';
import { createProcessStep } from '../process_step';
@ -14,8 +15,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
onClose: () => void;
}> = ({ onClose }) => {
const {
categoryQualityWarnings,
cleanUpAndSetUp,
endTime,
fetchJobStatus,
isValidating,
lastSetupErrorMessages,
moduleDescriptor,
@ -30,6 +33,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
viewResults,
} = useLogEntryCategoriesSetup();
useMount(() => {
fetchJobStatus();
});
const viewResultsAndClose = useCallback(() => {
viewResults();
onClose();
@ -47,6 +54,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
setupStatus,
setValidatedIndices,
validationErrors,
previousQualityWarnings: categoryQualityWarnings,
}),
createProcessStep({
cleanUpAndSetUp,
@ -58,6 +66,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
}),
],
[
categoryQualityWarnings,
cleanUpAndSetUp,
endTime,
isValidating,

68
x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout.tsx
View file

@ -15,14 +15,16 @@ import {
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import React from 'react';
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
import { LogEntryCategoriesSetupView } from './log_entry_categories_setup_view';
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
import { LogAnalysisModuleList } from './module_list';
import { useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
import { ModuleId, moduleIds, useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
const FLYOUT_HEADING_ID = 'logAnalysisSetupFlyoutHeading';
export const LogAnalysisSetupFlyout: React.FC = () => {
export const LogAnalysisSetupFlyout: React.FC<{
allowedModules?: ModuleId[];
}> = ({ allowedModules = moduleIds }) => {
const {
closeFlyout,
flyoutView,
@ -49,32 +51,58 @@ export const LogAnalysisSetupFlyout: React.FC = () => {
<EuiFlyoutBody>
{flyoutView.view === 'moduleList' ? (
<LogAnalysisModuleList onViewModuleSetup={showModuleSetup} />
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_analysis' ? (
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
<LogEntryRateSetupView onClose={closeFlyout} />
</LogAnalysisSetupFlyoutSubPage>
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_categories' ? (
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
<LogEntryCategoriesSetupView onClose={closeFlyout} />
</LogAnalysisSetupFlyoutSubPage>
) : flyoutView.view === 'moduleSetup' && allowedModules.includes(flyoutView.module) ? (
<ModuleSetupView
moduleId={flyoutView.module}
onClose={closeFlyout}
onViewModuleList={allowedModules.length > 1 ? showModuleList : undefined}
/>
) : null}
</EuiFlyoutBody>
</EuiFlyout>
);
};
const ModuleSetupView: React.FC<{
moduleId: ModuleId;
onClose: () => void;
onViewModuleList?: () => void;
}> = ({ moduleId, onClose, onViewModuleList }) => {
switch (moduleId) {
case 'logs_ui_analysis':
return (
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
<LogEntryRateSetupView onClose={onClose} />
</LogAnalysisSetupFlyoutSubPage>
);
case 'logs_ui_categories':
return (
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
<LogEntryCategoriesSetupView onClose={onClose} />
</LogAnalysisSetupFlyoutSubPage>
);
}
};
const LogAnalysisSetupFlyoutSubPage: React.FC<{
onViewModuleList: () => void;
onViewModuleList?: () => void;
}> = ({ children, onViewModuleList }) => (
<EuiFlexGroup alignItems="flexStart" direction="column" gutterSize="none">
<EuiFlexItem grow={false}>
<EuiButtonEmpty flush="left" iconSide="left" iconType="arrowLeft" onClick={onViewModuleList}>
<FormattedMessage
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
defaultMessage="All Machine Learning jobs"
/>
</EuiButtonEmpty>
</EuiFlexItem>
{onViewModuleList ? (
<EuiFlexItem grow={false}>
<EuiButtonEmpty
flush="left"
iconSide="left"
iconType="arrowLeft"
onClick={onViewModuleList}
>
<FormattedMessage
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
defaultMessage="All Machine Learning jobs"
/>
</EuiButtonEmpty>
</EuiFlexItem>
) : null}
<EuiFlexItem>{children}</EuiFlexItem>
</EuiFlexGroup>
);

2
x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_flyout/setup_flyout_state.ts
View file

@ -9,6 +9,8 @@ import { useState, useCallback } from 'react';
export type ModuleId = 'logs_ui_analysis' | 'logs_ui_categories';
export const moduleIds = ['logs_ui_analysis', 'logs_ui_categories'] as const;
type FlyoutView =
| { view: 'hidden' }
| { view: 'moduleList' }

47
x-pack/plugins/infra/public/containers/logs/log_analysis/api/get_latest_categories_datasets_stats.ts Normal file
View file

@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { HttpHandler } from 'src/core/public';
import {
CategorizerStatus,
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
LogEntryCategoriesDatasetStats,
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
} from '../../../../../common/http_api';
import { decodeOrThrow } from '../../../../../common/runtime_types';
export { LogEntryCategoriesDatasetStats };
export const callGetLatestCategoriesDatasetsStatsAPI = async (
{
jobIds,
startTime,
endTime,
includeCategorizerStatuses,
}: {
jobIds: string[];
startTime: number;
endTime: number;
includeCategorizerStatuses: CategorizerStatus[];
},
fetch: HttpHandler
) => {
const response = await fetch(LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, {
method: 'POST',
body: JSON.stringify(
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT.encode({
data: {
jobIds,
timeRange: { startTime, endTime },
includeCategorizerStatuses,
},
})
),
});
return decodeOrThrow(getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT)(response);
};

14
x-pack/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts
View file

@ -54,6 +54,17 @@ const jobStateRT = rt.keyof({
opening: null,
});
const jobAnalysisConfigRT = rt.partial({
per_partition_categorization: rt.intersection([
rt.type({
enabled: rt.boolean,
}),
rt.partial({
stop_on_warn: rt.boolean,
}),
]),
});
const jobCategorizationStatusRT = rt.keyof({
ok: null,
warn: null,
@ -64,6 +75,7 @@ const jobModelSizeStatsRT = rt.type({
categorized_doc_count: rt.number,
dead_category_count: rt.number,
frequent_category_count: rt.number,
log_time: rt.number,
rare_category_count: rt.number,
total_category_count: rt.number,
});
@ -79,6 +91,8 @@ export const jobSummaryRT = rt.intersection([
datafeedIndices: rt.array(rt.string),
datafeedState: datafeedStateRT,
fullJob: rt.partial({
analysis_config: jobAnalysisConfigRT,
create_time: rt.number,
custom_settings: jobCustomSettingsRT,
finished_time: rt.number,
model_size_stats: jobModelSizeStatsRT,

40
x-pack/plugins/infra/public/containers/logs/log_analysis/log_analysis_module_types.ts
View file

@ -50,43 +50,3 @@ export interface ModuleSourceConfiguration {
spaceId: string;
timestampField: string;
}
interface ManyCategoriesWarningReason {
type: 'manyCategories';
categoriesDocumentRatio: number;
}
interface ManyDeadCategoriesWarningReason {
type: 'manyDeadCategories';
deadCategoriesRatio: number;
}
interface ManyRareCategoriesWarningReason {
type: 'manyRareCategories';
rareCategoriesRatio: number;
}
interface NoFrequentCategoriesWarningReason {
type: 'noFrequentCategories';
}
interface SingleCategoryWarningReason {
type: 'singleCategory';
}
export type CategoryQualityWarningReason =
| ManyCategoriesWarningReason
| ManyDeadCategoriesWarningReason
| ManyRareCategoriesWarningReason
| NoFrequentCategoriesWarningReason
| SingleCategoryWarningReason;
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
export interface CategoryQualityWarning {
type: 'categoryQualityWarning';
jobId: string;
reasons: CategoryQualityWarningReason[];
}
export type QualityWarning = CategoryQualityWarning;

107
x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_quality.ts
View file

@ -4,43 +4,124 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { useMemo } from 'react';
import { useMemo, useState } from 'react';
import { useDeepCompareEffect } from 'react-use';
import {
JobModelSizeStats,
JobSummary,
QualityWarning,
CategoryQualityWarningReason,
} from '../../log_analysis_module_types';
QualityWarning,
} from '../../../../../../common/log_analysis';
import { useKibanaContextForPlugin } from '../../../../../hooks/use_kibana';
import { useTrackedPromise } from '../../../../../utils/use_tracked_promise';
import {
callGetLatestCategoriesDatasetsStatsAPI,
LogEntryCategoriesDatasetStats,
} from '../../api/get_latest_categories_datasets_stats';
import { JobModelSizeStats, JobSummary } from '../../log_analysis_module_types';
export const useLogEntryCategoriesQuality = ({ jobSummaries }: { jobSummaries: JobSummary[] }) => {
const {
services: {
http: { fetch },
},
} = useKibanaContextForPlugin();
const [lastestWarnedDatasetsStats, setLatestWarnedDatasetsStats] = useState<
LogEntryCategoriesDatasetStats[]
>([]);
const jobSummariesWithCategoryWarnings = useMemo(
() => jobSummaries.filter(isJobWithCategoryWarnings),
[jobSummaries]
);
const jobSummariesWithPartitionedCategoryWarnings = useMemo(
() => jobSummariesWithCategoryWarnings.filter(isJobWithPartitionedCategories),
[jobSummariesWithCategoryWarnings]
);
const [fetchLatestWarnedDatasetsStatsRequest, fetchLatestWarnedDatasetsStats] = useTrackedPromise(
{
cancelPreviousOn: 'creation',
createPromise: (
statsIntervals: Array<{ jobId: string; startTime: number; endTime: number }>
) =>
Promise.all(
statsIntervals.map(({ jobId, startTime, endTime }) =>
callGetLatestCategoriesDatasetsStatsAPI(
{ jobIds: [jobId], startTime, endTime, includeCategorizerStatuses: ['warn'] },
fetch
)
)
),
onResolve: (results) => {
setLatestWarnedDatasetsStats(results.flatMap(({ data: { datasetStats } }) => datasetStats));
},
},
[]
);
useDeepCompareEffect(() => {
fetchLatestWarnedDatasetsStats(
jobSummariesWithPartitionedCategoryWarnings.map((jobSummary) => ({
jobId: jobSummary.id,
startTime: jobSummary.fullJob?.create_time ?? 0,
endTime: jobSummary.fullJob?.model_size_stats?.log_time ?? Date.now(),
}))
);
}, [jobSummariesWithPartitionedCategoryWarnings]);
const categoryQualityWarnings: QualityWarning[] = useMemo(
() =>
jobSummaries
.filter(
(jobSummary) => jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn'
)
() => [
...jobSummariesWithCategoryWarnings
.filter((jobSummary) => !isJobWithPartitionedCategories(jobSummary))
.map((jobSummary) => ({
type: 'categoryQualityWarning',
type: 'categoryQualityWarning' as const,
jobId: jobSummary.id,
dataset: '',
reasons: jobSummary.fullJob?.model_size_stats
? getCategoryQualityWarningReasons(jobSummary.fullJob.model_size_stats)
: [],
})),
[jobSummaries]
...lastestWarnedDatasetsStats.map((datasetStats) => ({
type: 'categoryQualityWarning' as const,
jobId: datasetStats.job_id,
dataset: datasetStats.dataset,
reasons: getCategoryQualityWarningReasons(datasetStats),
})),
],
[jobSummariesWithCategoryWarnings, lastestWarnedDatasetsStats]
);
return {
categoryQualityWarnings,
lastLatestWarnedDatasetsStatsRequestErrors:
fetchLatestWarnedDatasetsStatsRequest.state === 'rejected'
? fetchLatestWarnedDatasetsStatsRequest.value
: null,
isLoadingCategoryQualityWarnings: fetchLatestWarnedDatasetsStatsRequest.state === 'pending',
};
};
const isJobWithCategoryWarnings = (jobSummary: JobSummary) =>
jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn';
const isJobWithPartitionedCategories = (jobSummary: JobSummary) =>
jobSummary.fullJob?.analysis_config?.per_partition_categorization ?? false;
const getCategoryQualityWarningReasons = ({
categorized_doc_count: categorizedDocCount,
dead_category_count: deadCategoryCount,
frequent_category_count: frequentCategoryCount,
rare_category_count: rareCategoryCount,
total_category_count: totalCategoryCount,
}: JobModelSizeStats): CategoryQualityWarningReason[] => {
}: Pick<
JobModelSizeStats,
| 'categorized_doc_count'
| 'dead_category_count'
| 'frequent_category_count'
| 'rare_category_count'
| 'total_category_count'
>): CategoryQualityWarningReason[] => {
const rareCategoriesRatio = rareCategoryCount / totalCategoryCount;
const categoriesDocumentRatio = totalCategoryCount / categorizedDocCount;
const deadCategoriesRatio = deadCategoryCount / totalCategoryCount;

4
x-pack/plugins/infra/public/containers/logs/log_analysis/modules/log_entry_categories/use_log_entry_categories_setup.tsx
View file

@ -9,7 +9,9 @@ import { useLogEntryCategoriesModuleContext } from './use_log_entry_categories_m
export const useLogEntryCategoriesSetup = () => {
const {
categoryQualityWarnings,
cleanUpAndSetUpModule,
fetchJobStatus,
lastSetupErrorMessages,
moduleDescriptor,
setUpModule,
@ -37,8 +39,10 @@ export const useLogEntryCategoriesSetup = () => {
});
return {
categoryQualityWarnings,
cleanUpAndSetUp,
endTime,
fetchJobStatus,
isValidating,
lastSetupErrorMessages,
moduleDescriptor,

24
x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_content.tsx
View file

@ -5,7 +5,7 @@
*/
import { i18n } from '@kbn/i18n';
import React, { useCallback, useEffect, useState } from 'react';
import React, { useCallback, useEffect } from 'react';
import { isJobStatusWithResults } from '../../../../common/log_analysis';
import { LoadingPage } from '../../../components/loading_page';
import {
@ -14,6 +14,10 @@ import {
MissingSetupPrivilegesPrompt,
SubscriptionSplashContent,
} from '../../../components/logging/log_analysis_setup';
import {
LogAnalysisSetupFlyout,
useLogAnalysisSetupFlyoutStateContext,
} from '../../../components/logging/log_analysis_setup/setup_flyout';
import { SourceErrorPage } from '../../../components/source_error_page';
import { SourceLoadingPage } from '../../../components/source_loading_page';
import { useLogAnalysisCapabilitiesContext } from '../../../containers/logs/log_analysis';
@ -21,7 +25,6 @@ import { useLogEntryCategoriesModuleContext } from '../../../containers/logs/log
import { useLogSourceContext } from '../../../containers/logs/log_source';
import { LogEntryCategoriesResultsContent } from './page_results_content';
import { LogEntryCategoriesSetupContent } from './page_setup_content';
import { LogEntryCategoriesSetupFlyout } from './setup_flyout';
export const LogEntryCategoriesPageContent = () => {
const {
@ -40,9 +43,10 @@ export const LogEntryCategoriesPageContent = () => {
const { fetchJobStatus, setupStatus, jobStatus } = useLogEntryCategoriesModuleContext();
const [isFlyoutOpen, setIsFlyoutOpen] = useState<boolean>(false);
const openFlyout = useCallback(() => setIsFlyoutOpen(true), []);
const closeFlyout = useCallback(() => setIsFlyoutOpen(false), []);
const { showModuleSetup } = useLogAnalysisSetupFlyoutStateContext();
const showCategoriesModuleSetup = useCallback(() => showModuleSetup('logs_ui_categories'), [
showModuleSetup,
]);
useEffect(() => {
if (hasLogAnalysisReadCapabilities) {
@ -71,8 +75,8 @@ export const LogEntryCategoriesPageContent = () => {
} else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) {
return (
<>
<LogEntryCategoriesResultsContent onOpenSetup={openFlyout} />
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
<LogEntryCategoriesResultsContent onOpenSetup={showCategoriesModuleSetup} />
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
</>
);
} else if (!hasLogAnalysisSetupCapabilities) {
@ -80,9 +84,11 @@ export const LogEntryCategoriesPageContent = () => {
} else {
return (
<>
<LogEntryCategoriesSetupContent onOpenSetup={openFlyout} />
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
<LogEntryCategoriesSetupContent onOpenSetup={showCategoriesModuleSetup} />
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
</>
);
}
};
const allowedSetupModules = ['logs_ui_categories' as const];

3
x-pack/plugins/infra/public/pages/logs/log_entry_categories/page_providers.tsx
View file

@ -5,6 +5,7 @@
*/
import React from 'react';
import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout';
import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
import { useLogSourceContext } from '../../../containers/logs/log_source';
import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
@ -27,7 +28,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child
spaceId={space.id}
timestampField={sourceConfiguration.configuration.fields.timestamp}
>
{children}
<LogAnalysisSetupFlyoutStateProvider>{children}</LogAnalysisSetupFlyoutStateProvider>
</LogEntryCategoriesModuleProvider>
);
};

128
x-pack/plugins/infra/public/pages/logs/log_entry_categories/setup_flyout.tsx
View file

@ -1,128 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
EuiFlyout,
EuiFlyoutBody,
EuiFlyoutHeader,
EuiSpacer,
EuiSteps,
EuiText,
EuiTitle,
} from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { useCallback, useMemo } from 'react';
import {
createInitialConfigurationStep,
createProcessStep,
} from '../../../components/logging/log_analysis_setup';
import { useLogEntryCategoriesSetup } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
interface LogEntryCategoriesSetupFlyoutProps {
isOpen: boolean;
onClose: () => void;
}
export const LogEntryCategoriesSetupFlyout: React.FC<LogEntryCategoriesSetupFlyoutProps> = ({
isOpen,
onClose,
}) => {
const {
cleanUpAndSetUp,
endTime,
isValidating,
lastSetupErrorMessages,
setEndTime,
setStartTime,
setValidatedIndices,
setUp,
setupStatus,
startTime,
validatedIndices,
validationErrors,
viewResults,
} = useLogEntryCategoriesSetup();
const viewResultsAndClose = useCallback(() => {
viewResults();
onClose();
}, [viewResults, onClose]);
const steps = useMemo(
() => [
createInitialConfigurationStep({
setStartTime,
setEndTime,
startTime,
endTime,
isValidating,
validatedIndices,
setupStatus,
setValidatedIndices,
validationErrors,
}),
createProcessStep({
cleanUpAndSetUp,
errorMessages: lastSetupErrorMessages,
isConfigurationValid: validationErrors.length <= 0 && !isValidating,
setUp,
setupStatus,
viewResults: viewResultsAndClose,
}),
],
[
cleanUpAndSetUp,
endTime,
isValidating,
lastSetupErrorMessages,
setEndTime,
setStartTime,
setUp,
setValidatedIndices,
setupStatus,
startTime,
validatedIndices,
validationErrors,
viewResultsAndClose,
]
);
if (!isOpen) {
return null;
}
return (
<EuiFlyout onClose={onClose}>
<EuiFlyoutHeader hasBorder>
<EuiTitle size="s">
<h3>
<FormattedMessage
id="xpack.infra.logs.setupFlyout.setupFlyoutTitle"
defaultMessage="Anomaly detection with Machine Learning"
/>
</h3>
</EuiTitle>
</EuiFlyoutHeader>
<EuiFlyoutBody>
<EuiTitle size="s">
<h3>
<FormattedMessage
id="xpack.infra.logs.setupFlyout.logCategoriesTitle"
defaultMessage="Log categories"
/>
</h3>
</EuiTitle>
<EuiText size="s">
<FormattedMessage
id="xpack.infra.logs.setupFlyout.logCategoriesDescription"
defaultMessage="Use Machine Learning to automatically categorize log messages."
/>
</EuiText>
<EuiSpacer />
<EuiSteps steps={steps} />
</EuiFlyoutBody>
</EuiFlyout>
);
};

2
x-pack/plugins/infra/server/infra_server.ts
View file

@ -13,6 +13,7 @@ import { InfraBackendLibs } from './lib/infra_types';
import {
initGetLogEntryCategoriesRoute,
initGetLogEntryCategoryDatasetsRoute,
initGetLogEntryCategoryDatasetsStatsRoute,
initGetLogEntryCategoryExamplesRoute,
initGetLogEntryRateRoute,
initGetLogEntryExamplesRoute,
@ -54,6 +55,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
initIpToHostName(libs);
initGetLogEntryCategoriesRoute(libs);
initGetLogEntryCategoryDatasetsRoute(libs);
initGetLogEntryCategoryDatasetsStatsRoute(libs);
initGetLogEntryCategoryExamplesRoute(libs);
initGetLogEntryRateRoute(libs);
initGetLogEntryAnomaliesRoute(libs);

2
x-pack/plugins/infra/server/lib/log_analysis/common.ts
View file

@ -36,7 +36,7 @@ export async function fetchMlJob(mlAnomalyDetectors: MlAnomalyDetectors, jobId:
};
}
const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
export const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
// Finds datasets related to ML job ids
export async function getLogEntryDatasets(

1
x-pack/plugins/infra/server/lib/log_analysis/index.ts
View file

@ -6,5 +6,6 @@
export * from './errors';
export * from './log_entry_categories_analysis';
export * from './log_entry_categories_datasets_stats';
export * from './log_entry_rate_analysis';
export * from './log_entry_anomalies';

94
x-pack/plugins/infra/server/lib/log_analysis/log_entry_categories_datasets_stats.ts Normal file
View file

@ -0,0 +1,94 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { startTracingSpan } from '../../../common/performance_tracing';
import { decodeOrThrow } from '../../../common/runtime_types';
import type { MlAnomalyDetectors, MlSystem } from '../../types';
import { COMPOSITE_AGGREGATION_BATCH_SIZE } from './common';
import {
CompositeDatasetKey,
createLatestLogEntryCategoriesDatasetsStatsQuery,
latestLogEntryCategoriesDatasetsStatsResponseRT,
LogEntryCategoryDatasetStatsBucket,
} from './queries/latest_log_entry_categories_datasets_stats';
export async function getLatestLogEntriesCategoriesDatasetsStats(
context: {
infra: {
mlAnomalyDetectors: MlAnomalyDetectors;
mlSystem: MlSystem;
};
},
jobIds: string[],
startTime: number,
endTime: number,
includeCategorizerStatuses: Array<'ok' | 'warn'> = []
) {
const finalizeLogEntryCategoriesDatasetsStats = startTracingSpan('get categories datasets stats');
let latestLogEntryCategoriesDatasetsStatsBuckets: LogEntryCategoryDatasetStatsBucket[] = [];
let afterLatestBatchKey: CompositeDatasetKey | undefined;
while (true) {
const latestLogEntryCategoriesDatasetsStatsResponse = await context.infra.mlSystem.mlAnomalySearch(
createLatestLogEntryCategoriesDatasetsStatsQuery(
jobIds,
startTime,
endTime,
COMPOSITE_AGGREGATION_BATCH_SIZE,
afterLatestBatchKey
)
);
const { after_key: afterKey, buckets: latestBatchBuckets = [] } =
decodeOrThrow(latestLogEntryCategoriesDatasetsStatsResponseRT)(
latestLogEntryCategoriesDatasetsStatsResponse
).aggregations?.dataset_composite_terms ?? {};
const latestIncludedBatchBuckets =
includeCategorizerStatuses.length > 0
? latestBatchBuckets.filter((bucket) =>
bucket.categorizer_stats_top_hits.hits.hits.some((hit) =>
includeCategorizerStatuses.includes(hit._source.categorization_status)
)
)
: latestBatchBuckets;
latestLogEntryCategoriesDatasetsStatsBuckets = [
...latestLogEntryCategoriesDatasetsStatsBuckets,
...latestIncludedBatchBuckets,
];
afterLatestBatchKey = afterKey;
if (afterKey == null || latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) {
break;
}
}
const logEntryCategoriesDatasetsStatsSpan = finalizeLogEntryCategoriesDatasetsStats();
return {
data: latestLogEntryCategoriesDatasetsStatsBuckets.map((bucket) => {
const latestHitSource = bucket.categorizer_stats_top_hits.hits.hits[0]._source;
return {
categorization_status: latestHitSource.categorization_status,
categorized_doc_count: latestHitSource.categorized_doc_count,
dataset: bucket.key.dataset ?? '',
dead_category_count: latestHitSource.dead_category_count,
failed_category_count: latestHitSource.failed_category_count,
frequent_category_count: latestHitSource.frequent_category_count,
job_id: latestHitSource.job_id,
log_time: latestHitSource.log_time,
rare_category_count: latestHitSource.rare_category_count,
total_category_count: latestHitSource.total_category_count,
};
}),
timing: {
spans: [logEntryCategoriesDatasetsStatsSpan],
},
};
}

15
x-pack/plugins/infra/server/lib/log_analysis/queries/common.ts
View file

@ -40,7 +40,20 @@ export const createTimeRangeFilters = (startTime: number, endTime: number) => [
},
];
export const createResultTypeFilters = (resultTypes: Array<'model_plot' | 'record'>) => [
export const createLogTimeRangeFilters = (startTime: number, endTime: number) => [
{
range: {
log_time: {
gte: startTime,
lte: endTime,
},
},
},
];
export const createResultTypeFilters = (
resultTypes: Array<'categorizer_stats' | 'model_plot' | 'record'>
) => [
{
terms: {
result_type: resultTypes,

133
x-pack/plugins/infra/server/lib/log_analysis/queries/latest_log_entry_categories_datasets_stats.ts Normal file
View file

@ -0,0 +1,133 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import * as rt from 'io-ts';
import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types';
import {
createJobIdsFilters,
createResultTypeFilters,
defaultRequestParameters,
createLogTimeRangeFilters,
} from './common';
export const createLatestLogEntryCategoriesDatasetsStatsQuery = (
logEntryCategoriesJobIds: string[],
startTime: number,
endTime: number,
size: number,
afterKey?: CompositeDatasetKey
) => ({
...defaultRequestParameters,
body: {
query: {
bool: {
filter: [
...createJobIdsFilters(logEntryCategoriesJobIds),
...createResultTypeFilters(['categorizer_stats']),
...createLogTimeRangeFilters(startTime, endTime),
],
},
},
aggregations: {
dataset_composite_terms: {
composite: {
after: afterKey,
size,
sources: [
{
dataset: {
terms: {
field: 'partition_field_value',
missing_bucket: true,
},
},
},
],
},
aggs: {
categorizer_stats_top_hits: {
top_hits: {
size: 1,
sort: [
{
log_time: 'desc',
},
],
_source: [
'categorization_status',
'categorized_doc_count',
'dead_category_count',
'failed_category_count',
'frequent_category_count',
'job_id',
'log_time',
'rare_category_count',
'total_category_count',
],
},
},
},
},
},
},
size: 0,
});
export const logEntryCategoryStatusRT = rt.keyof({
ok: null,
warn: null,
});
export const logEntryCategorizerStatsHitRT = rt.type({
_source: rt.type({
categorization_status: logEntryCategoryStatusRT,
categorized_doc_count: rt.number,
dead_category_count: rt.number,
failed_category_count: rt.number,
frequent_category_count: rt.number,
job_id: rt.string,
log_time: rt.number,
rare_category_count: rt.number,
total_category_count: rt.number,
}),
});
export type LogEntryCategorizerStatsHit = rt.TypeOf<typeof logEntryCategorizerStatsHitRT>;
const compositeDatasetKeyRT = rt.type({
dataset: rt.union([rt.string, rt.null]),
});
export type CompositeDatasetKey = rt.TypeOf<typeof compositeDatasetKeyRT>;
const logEntryCategoryDatasetStatsBucketRT = rt.type({
key: compositeDatasetKeyRT,
categorizer_stats_top_hits: rt.type({
hits: rt.type({
hits: rt.array(logEntryCategorizerStatsHitRT),
}),
}),
});
export type LogEntryCategoryDatasetStatsBucket = rt.TypeOf<
typeof logEntryCategoryDatasetStatsBucketRT
>;
export const latestLogEntryCategoriesDatasetsStatsResponseRT = rt.intersection([
commonSearchSuccessResponseFieldsRT,
rt.partial({
aggregations: rt.type({
dataset_composite_terms: rt.type({
after_key: compositeDatasetKeyRT,
buckets: rt.array(logEntryCategoryDatasetStatsBucketRT),
}),
}),
}),
]);
export type LatestLogEntryCategoriesDatasetsStatsResponse = rt.TypeOf<
typeof latestLogEntryCategoriesDatasetsStatsResponseRT
>;

1
x-pack/plugins/infra/server/routes/log_analysis/results/index.ts
View file

@ -6,6 +6,7 @@
export * from './log_entry_categories';
export * from './log_entry_category_datasets';
export * from './log_entry_category_datasets_stats';
export * from './log_entry_category_examples';
export * from './log_entry_rate';
export * from './log_entry_examples';

79
x-pack/plugins/infra/server/routes/log_analysis/results/log_entry_category_datasets_stats.ts Normal file
View file

@ -0,0 +1,79 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Boom from 'boom';
import {
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
} from '../../../../common/http_api/log_analysis';
import { createValidationFunction } from '../../../../common/runtime_types';
import type { InfraBackendLibs } from '../../../lib/infra_types';
import { getLatestLogEntriesCategoriesDatasetsStats } from '../../../lib/log_analysis';
import { isMlPrivilegesError } from '../../../lib/log_analysis/errors';
import { assertHasInfraMlPlugins } from '../../../utils/request_context';
export const initGetLogEntryCategoryDatasetsStatsRoute = ({ framework }: InfraBackendLibs) => {
framework.registerRoute(
{
method: 'post',
path: LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
validate: {
body: createValidationFunction(getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT),
},
},
framework.router.handleLegacyErrors(async (requestContext, request, response) => {
const {
data: {
jobIds,
timeRange: { startTime, endTime },
includeCategorizerStatuses,
},
} = request.body;
try {
assertHasInfraMlPlugins(requestContext);
const { data: datasetStats, timing } = await getLatestLogEntriesCategoriesDatasetsStats(
requestContext,
jobIds,
startTime,
endTime,
includeCategorizerStatuses
);
return response.ok({
body: getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT.encode({
data: {
datasetStats,
},
timing,
}),
});
} catch (error) {
if (Boom.isBoom(error)) {
throw error;
}
if (isMlPrivilegesError(error)) {
return response.customError({
statusCode: 403,
body: {
message: error.message,
},
});
}
return response.customError({
statusCode: error.statusCode ?? 500,
body: {
message: error.message ?? 'An unexpected error occurred',
},
});
}
})
);
};

8
x-pack/plugins/ml/server/models/data_recognizer/modules/logs_ui_categories/ml/log_entry_categories_count.json
View file

@ -14,7 +14,11 @@
"use_null": true
}
],
"influencers": ["event.dataset", "mlcategory"]
"influencers": ["event.dataset", "mlcategory"],
"per_partition_categorization": {
"enabled": true,
"stop_on_warn": false
}
},
"analysis_limits": {
"model_memory_limit": "100mb",
@ -29,6 +33,6 @@
},
"custom_settings": {
"created_by": "ml-module-logs-ui-categories",
"job_revision": 0
"job_revision": 1
}
}

3
x-pack/plugins/translations/translations/ja-JP.json
View file

@ -8486,9 +8486,6 @@
"xpack.infra.logs.search.searchInLogsAriaLabel": "検索",
"xpack.infra.logs.search.searchInLogsPlaceholder": "検索",
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 件のハイライトされたエントリー} other {# 件のハイライトされたエントリー}}",
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "機械学習を使用して、ログメッセージを自動的に分類します。",
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "ログカテゴリー",
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "機械学習を使用した異常検知",
"xpack.infra.logs.showingEntriesFromTimestamp": "{timestamp} 以降のエントリーを表示中",
"xpack.infra.logs.showingEntriesUntilTimestamp": "{timestamp} までのエントリーを表示中",
"xpack.infra.logs.startStreamingButtonLabel": "ライブストリーム",

3
x-pack/plugins/translations/translations/zh-CN.json
View file

@ -8491,9 +8491,6 @@
"xpack.infra.logs.search.searchInLogsAriaLabel": "搜索",
"xpack.infra.logs.search.searchInLogsPlaceholder": "搜索",
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 个高亮条目} other {# 个高亮条目}}",
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "使用 Machine Learning 自动归类日志消息。",
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "日志类别",
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "通过 Machine Learning 检测异常",
"xpack.infra.logs.showingEntriesFromTimestamp": "正在显示自 {timestamp} 起的条目",
"xpack.infra.logs.showingEntriesUntilTimestamp": "正在显示截止于 {timestamp} 的条目",
"xpack.infra.logs.startStreamingButtonLabel": "实时流式传输",