[Logs UI] Add dataset-specific categorization warnings (#75351)
This adds dataset-specific categorization warnings for the categorization module. The warnings are displayed in call-outs on the relevant tabs as well as the job setup screens if a prior job with warnings exists. To that end this also changes the categorization job configuration to enable the partitioned categorization mode. Co-authored-by: Alejandro Fernández Gómez <antarticonorte@gmail.com>
This commit is contained in:
parent
5ff0c00529
commit
3f2e9f7705
|
@ -23,7 +23,7 @@ export const storybookAliases = {
|
||||||
codeeditor: 'src/plugins/kibana_react/public/code_editor/scripts/storybook.ts',
|
codeeditor: 'src/plugins/kibana_react/public/code_editor/scripts/storybook.ts',
|
||||||
dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/scripts/storybook.js',
|
dashboard_enhanced: 'x-pack/plugins/dashboard_enhanced/scripts/storybook.js',
|
||||||
embeddable: 'src/plugins/embeddable/scripts/storybook.js',
|
embeddable: 'src/plugins/embeddable/scripts/storybook.js',
|
||||||
infra: 'x-pack/legacy/plugins/infra/scripts/storybook.js',
|
infra: 'x-pack/plugins/infra/scripts/storybook.js',
|
||||||
security_solution: 'x-pack/plugins/security_solution/scripts/storybook.js',
|
security_solution: 'x-pack/plugins/security_solution/scripts/storybook.js',
|
||||||
ui_actions_enhanced: 'x-pack/plugins/ui_actions_enhanced/scripts/storybook.js',
|
ui_actions_enhanced: 'x-pack/plugins/ui_actions_enhanced/scripts/storybook.js',
|
||||||
observability: 'x-pack/plugins/observability/scripts/storybook.js',
|
observability: 'x-pack/plugins/observability/scripts/storybook.js',
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
export * from './log_entry_categories';
|
export * from './log_entry_categories';
|
||||||
export * from './log_entry_category_datasets';
|
export * from './log_entry_category_datasets';
|
||||||
|
export * from './log_entry_category_datasets_stats';
|
||||||
export * from './log_entry_category_examples';
|
export * from './log_entry_category_examples';
|
||||||
export * from './log_entry_rate';
|
export * from './log_entry_rate';
|
||||||
export * from './log_entry_examples';
|
export * from './log_entry_examples';
|
||||||
|
|
|
@ -0,0 +1,72 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as rt from 'io-ts';
|
||||||
|
|
||||||
|
import { timeRangeRT, routeTimingMetadataRT } from '../../shared';
|
||||||
|
|
||||||
|
export const LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH =
|
||||||
|
'/api/infra/log_analysis/results/latest_log_entry_category_datasets_stats';
|
||||||
|
|
||||||
|
const categorizerStatusRT = rt.keyof({
|
||||||
|
ok: null,
|
||||||
|
warn: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
export type CategorizerStatus = rt.TypeOf<typeof categorizerStatusRT>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* request
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT = rt.type({
|
||||||
|
data: rt.type({
|
||||||
|
// the ids of the categorization jobs
|
||||||
|
jobIds: rt.array(rt.string),
|
||||||
|
// the time range to fetch the category datasets stats for
|
||||||
|
timeRange: timeRangeRT,
|
||||||
|
// the categorizer statuses to include stats for, empty means all
|
||||||
|
includeCategorizerStatuses: rt.array(categorizerStatusRT),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type GetLatestLogEntryCategoryDatasetsStatsRequestPayload = rt.TypeOf<
|
||||||
|
typeof getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT
|
||||||
|
>;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* response
|
||||||
|
*/
|
||||||
|
|
||||||
|
const logEntryCategoriesDatasetStatsRT = rt.type({
|
||||||
|
categorization_status: categorizerStatusRT,
|
||||||
|
categorized_doc_count: rt.number,
|
||||||
|
dataset: rt.string,
|
||||||
|
dead_category_count: rt.number,
|
||||||
|
failed_category_count: rt.number,
|
||||||
|
frequent_category_count: rt.number,
|
||||||
|
job_id: rt.string,
|
||||||
|
log_time: rt.number,
|
||||||
|
rare_category_count: rt.number,
|
||||||
|
total_category_count: rt.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
export type LogEntryCategoriesDatasetStats = rt.TypeOf<typeof logEntryCategoriesDatasetStatsRT>;
|
||||||
|
|
||||||
|
export const getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT = rt.intersection([
|
||||||
|
rt.type({
|
||||||
|
data: rt.type({
|
||||||
|
datasetStats: rt.array(logEntryCategoriesDatasetStatsRT),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
rt.partial({
|
||||||
|
timing: routeTimingMetadataRT,
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
export type GetLatestLogEntryCategoryDatasetsStatsSuccessResponsePayload = rt.TypeOf<
|
||||||
|
typeof getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT
|
||||||
|
>;
|
|
@ -5,6 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export * from './log_analysis';
|
export * from './log_analysis';
|
||||||
|
export * from './log_analysis_quality';
|
||||||
export * from './log_analysis_results';
|
export * from './log_analysis_results';
|
||||||
export * from './log_entry_rate_analysis';
|
export * from './log_entry_rate_analysis';
|
||||||
export * from './log_entry_categories_analysis';
|
export * from './log_entry_categories_analysis';
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
interface ManyCategoriesWarningReason {
|
||||||
|
type: 'manyCategories';
|
||||||
|
categoriesDocumentRatio: number;
|
||||||
|
}
|
||||||
|
interface ManyDeadCategoriesWarningReason {
|
||||||
|
type: 'manyDeadCategories';
|
||||||
|
deadCategoriesRatio: number;
|
||||||
|
}
|
||||||
|
interface ManyRareCategoriesWarningReason {
|
||||||
|
type: 'manyRareCategories';
|
||||||
|
rareCategoriesRatio: number;
|
||||||
|
}
|
||||||
|
interface NoFrequentCategoriesWarningReason {
|
||||||
|
type: 'noFrequentCategories';
|
||||||
|
}
|
||||||
|
interface SingleCategoryWarningReason {
|
||||||
|
type: 'singleCategory';
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CategoryQualityWarningReason =
|
||||||
|
| ManyCategoriesWarningReason
|
||||||
|
| ManyDeadCategoriesWarningReason
|
||||||
|
| ManyRareCategoriesWarningReason
|
||||||
|
| NoFrequentCategoriesWarningReason
|
||||||
|
| SingleCategoryWarningReason;
|
||||||
|
|
||||||
|
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
|
||||||
|
|
||||||
|
export interface CategoryQualityWarning {
|
||||||
|
type: 'categoryQualityWarning';
|
||||||
|
jobId: string;
|
||||||
|
dataset: string;
|
||||||
|
reasons: CategoryQualityWarningReason[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type QualityWarning = CategoryQualityWarning;
|
|
@ -31,6 +31,7 @@ export const JobConfigurationOutdatedCallout: React.FC<{
|
||||||
values={{
|
values={{
|
||||||
moduleName,
|
moduleName,
|
||||||
}}
|
}}
|
||||||
|
tagName="p"
|
||||||
/>
|
/>
|
||||||
</RecreateJobCallout>
|
</RecreateJobCallout>
|
||||||
);
|
);
|
||||||
|
|
|
@ -31,6 +31,7 @@ export const JobDefinitionOutdatedCallout: React.FC<{
|
||||||
values={{
|
values={{
|
||||||
moduleName,
|
moduleName,
|
||||||
}}
|
}}
|
||||||
|
tagName="p"
|
||||||
/>
|
/>
|
||||||
</RecreateJobCallout>
|
</RecreateJobCallout>
|
||||||
);
|
);
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { QualityWarning } from '../../../containers/logs/log_analysis/log_analysis_module_types';
|
import { QualityWarning } from '../../../../common/log_analysis';
|
||||||
import { LogAnalysisJobProblemIndicator } from './log_analysis_job_problem_indicator';
|
import { LogAnalysisJobProblemIndicator } from './log_analysis_job_problem_indicator';
|
||||||
import { CategoryQualityWarnings } from './quality_warning_notices';
|
import { CategoryQualityWarnings } from './quality_warning_notices';
|
||||||
|
|
||||||
|
@ -41,6 +41,10 @@ export const CategoryJobNoticesSection: React.FC<{
|
||||||
onRecreateMlJobForReconfiguration={onRecreateMlJobForReconfiguration}
|
onRecreateMlJobForReconfiguration={onRecreateMlJobForReconfiguration}
|
||||||
onRecreateMlJobForUpdate={onRecreateMlJobForUpdate}
|
onRecreateMlJobForUpdate={onRecreateMlJobForUpdate}
|
||||||
/>
|
/>
|
||||||
<CategoryQualityWarnings qualityWarnings={qualityWarnings} />
|
<CategoryQualityWarnings
|
||||||
|
hasSetupCapabilities={hasSetupCapabilities}
|
||||||
|
qualityWarnings={qualityWarnings}
|
||||||
|
onRecreateMlJob={onRecreateMlJobForReconfiguration}
|
||||||
|
/>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|
|
@ -0,0 +1,68 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { action } from '@storybook/addon-actions';
|
||||||
|
import { storiesOf } from '@storybook/react';
|
||||||
|
import React from 'react';
|
||||||
|
import { EuiThemeProvider } from '../../../../../observability/public';
|
||||||
|
import { QualityWarning } from '../../../../common/log_analysis';
|
||||||
|
import { CategoryQualityWarnings } from './quality_warning_notices';
|
||||||
|
|
||||||
|
storiesOf('infra/logAnalysis/CategoryQualityWarnings', module)
|
||||||
|
.addDecorator((renderStory) => <EuiThemeProvider>{renderStory()}</EuiThemeProvider>)
|
||||||
|
.add('Partitioned warnings', () => {
|
||||||
|
return (
|
||||||
|
<CategoryQualityWarnings
|
||||||
|
hasSetupCapabilities={true}
|
||||||
|
onRecreateMlJob={action('on-recreate-ml-job')}
|
||||||
|
qualityWarnings={partitionedQualityWarnings}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.add('Unpartitioned warnings', () => {
|
||||||
|
return (
|
||||||
|
<CategoryQualityWarnings
|
||||||
|
hasSetupCapabilities={true}
|
||||||
|
onRecreateMlJob={action('on-recreate-ml-job')}
|
||||||
|
qualityWarnings={unpartitionedQualityWarnings}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const partitionedQualityWarnings: QualityWarning[] = [
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'theMlJobId',
|
||||||
|
dataset: 'first.dataset',
|
||||||
|
reasons: [
|
||||||
|
{ type: 'singleCategory' },
|
||||||
|
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
|
||||||
|
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'theMlJobId',
|
||||||
|
dataset: 'second.dataset',
|
||||||
|
reasons: [
|
||||||
|
{ type: 'noFrequentCategories' },
|
||||||
|
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.7 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const unpartitionedQualityWarnings: QualityWarning[] = [
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'theMlJobId',
|
||||||
|
dataset: '',
|
||||||
|
reasons: [
|
||||||
|
{ type: 'singleCategory' },
|
||||||
|
{ type: 'manyRareCategories', rareCategoriesRatio: 0.95 },
|
||||||
|
{ type: 'manyCategories', categoriesDocumentRatio: 0.7 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
|
@ -4,43 +4,89 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { EuiCallOut } from '@elastic/eui';
|
import {
|
||||||
|
EuiAccordion,
|
||||||
|
EuiDescriptionList,
|
||||||
|
EuiDescriptionListDescription,
|
||||||
|
EuiDescriptionListTitle,
|
||||||
|
EuiSpacer,
|
||||||
|
htmlIdGenerator,
|
||||||
|
} from '@elastic/eui';
|
||||||
import { i18n } from '@kbn/i18n';
|
import { i18n } from '@kbn/i18n';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React from 'react';
|
import groupBy from 'lodash/groupBy';
|
||||||
import type {
|
import React, { Fragment, useState } from 'react';
|
||||||
|
import { euiStyled } from '../../../../../observability/public';
|
||||||
|
import {
|
||||||
|
CategoryQualityWarning,
|
||||||
CategoryQualityWarningReason,
|
CategoryQualityWarningReason,
|
||||||
QualityWarning,
|
getFriendlyNameForPartitionId,
|
||||||
} from '../../../containers/logs/log_analysis/log_analysis_module_types';
|
} from '../../../../common/log_analysis';
|
||||||
|
import { RecreateJobCallout } from './recreate_job_callout';
|
||||||
|
|
||||||
export const CategoryQualityWarnings: React.FC<{ qualityWarnings: QualityWarning[] }> = ({
|
export const CategoryQualityWarnings: React.FC<{
|
||||||
qualityWarnings,
|
hasSetupCapabilities: boolean;
|
||||||
}) => (
|
onRecreateMlJob: () => void;
|
||||||
<>
|
qualityWarnings: CategoryQualityWarning[];
|
||||||
{qualityWarnings.map((qualityWarning, qualityWarningIndex) => (
|
}> = ({ hasSetupCapabilities, onRecreateMlJob, qualityWarnings }) => {
|
||||||
<EuiCallOut
|
const [detailAccordionId] = useState(htmlIdGenerator()());
|
||||||
key={`${qualityWarningIndex}`}
|
|
||||||
title={categoryQualityWarningCalloutTitle}
|
const categoryQualityWarningsByJob = groupBy(qualityWarnings, 'jobId');
|
||||||
color="warning"
|
|
||||||
iconType="alert"
|
return (
|
||||||
>
|
<>
|
||||||
<p>
|
{Object.entries(categoryQualityWarningsByJob).map(([jobId, qualityWarningsForJob]) => (
|
||||||
|
<RecreateJobCallout
|
||||||
|
hasSetupCapabilities={hasSetupCapabilities}
|
||||||
|
key={`quality-warnings-callout-${jobId}`}
|
||||||
|
onRecreateMlJob={onRecreateMlJob}
|
||||||
|
title={categoryQualityWarningCalloutTitle}
|
||||||
|
>
|
||||||
<FormattedMessage
|
<FormattedMessage
|
||||||
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningCalloutMessage"
|
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningCalloutMessage"
|
||||||
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results."
|
defaultMessage="While analyzing the log messages we've detected some problems which might indicate a reduced quality of the categorization results. Consider excluding the respective datasets from the analysis."
|
||||||
|
tagName="p"
|
||||||
/>
|
/>
|
||||||
</p>
|
<EuiAccordion
|
||||||
<ul>
|
id={detailAccordionId}
|
||||||
{qualityWarning.reasons.map((reason, reasonIndex) => (
|
buttonContent={
|
||||||
<li key={`${reasonIndex}`}>
|
<FormattedMessage
|
||||||
<CategoryQualityWarningReasonDescription reason={reason} />
|
id="xpack.infra.logs.logEntryCategories.categoryQualityWarningDetailsAccordionButtonLabel"
|
||||||
</li>
|
defaultMessage="Details"
|
||||||
))}
|
/>
|
||||||
</ul>
|
}
|
||||||
</EuiCallOut>
|
paddingSize="m"
|
||||||
))}
|
>
|
||||||
</>
|
<EuiDescriptionList>
|
||||||
);
|
{qualityWarningsForJob.flatMap((qualityWarning) => (
|
||||||
|
<Fragment key={`item-${getFriendlyNameForPartitionId(qualityWarning.dataset)}`}>
|
||||||
|
<EuiDescriptionListTitle data-test-subj={`title-${qualityWarning.dataset}`}>
|
||||||
|
{getFriendlyNameForPartitionId(qualityWarning.dataset)}
|
||||||
|
</EuiDescriptionListTitle>
|
||||||
|
{qualityWarning.reasons.map((reason) => (
|
||||||
|
<QualityWarningReasonDescription
|
||||||
|
key={`description-${reason.type}-${qualityWarning.dataset}`}
|
||||||
|
data-test-subj={`description-${reason.type}-${qualityWarning.dataset}`}
|
||||||
|
>
|
||||||
|
<CategoryQualityWarningReasonDescription reason={reason} />
|
||||||
|
</QualityWarningReasonDescription>
|
||||||
|
))}
|
||||||
|
</Fragment>
|
||||||
|
))}
|
||||||
|
</EuiDescriptionList>
|
||||||
|
</EuiAccordion>
|
||||||
|
<EuiSpacer size="l" />
|
||||||
|
</RecreateJobCallout>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
const QualityWarningReasonDescription = euiStyled(EuiDescriptionListDescription)`
|
||||||
|
display: list-item;
|
||||||
|
list-style-type: disc;
|
||||||
|
margin-left: ${(props) => props.theme.eui.paddingSizes.m};
|
||||||
|
`;
|
||||||
|
|
||||||
const categoryQualityWarningCalloutTitle = i18n.translate(
|
const categoryQualityWarningCalloutTitle = i18n.translate(
|
||||||
'xpack.infra.logs.logEntryCategories.categoryQUalityWarningCalloutTitle',
|
'xpack.infra.logs.logEntryCategories.categoryQUalityWarningCalloutTitle',
|
||||||
|
@ -49,7 +95,7 @@ const categoryQualityWarningCalloutTitle = i18n.translate(
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
const CategoryQualityWarningReasonDescription: React.FC<{
|
export const CategoryQualityWarningReasonDescription: React.FC<{
|
||||||
reason: CategoryQualityWarningReason;
|
reason: CategoryQualityWarningReason;
|
||||||
}> = ({ reason }) => {
|
}> = ({ reason }) => {
|
||||||
switch (reason.type) {
|
switch (reason.type) {
|
||||||
|
@ -57,7 +103,7 @@ const CategoryQualityWarningReasonDescription: React.FC<{
|
||||||
return (
|
return (
|
||||||
<FormattedMessage
|
<FormattedMessage
|
||||||
id="xpack.infra.logs.logEntryCategories.singleCategoryWarningReasonDescription"
|
id="xpack.infra.logs.logEntryCategories.singleCategoryWarningReasonDescription"
|
||||||
defaultMessage="The analysis couldn't extract more than a single category from the log message."
|
defaultMessage="The analysis couldn't extract more than a single category from the log messages."
|
||||||
/>
|
/>
|
||||||
);
|
);
|
||||||
case 'manyRareCategories':
|
case 'manyRareCategories':
|
||||||
|
|
|
@ -14,7 +14,7 @@ export const RecreateJobCallout: React.FC<{
|
||||||
title?: React.ReactNode;
|
title?: React.ReactNode;
|
||||||
}> = ({ children, hasSetupCapabilities, onRecreateMlJob, title }) => (
|
}> = ({ children, hasSetupCapabilities, onRecreateMlJob, title }) => (
|
||||||
<EuiCallOut color="warning" iconType="alert" title={title}>
|
<EuiCallOut color="warning" iconType="alert" title={title}>
|
||||||
<p>{children}</p>
|
{children}
|
||||||
<RecreateJobButton
|
<RecreateJobButton
|
||||||
color="warning"
|
color="warning"
|
||||||
hasSetupCapabilities={hasSetupCapabilities}
|
hasSetupCapabilities={hasSetupCapabilities}
|
||||||
|
|
|
@ -8,6 +8,7 @@ import { EuiDescribedFormGroup, EuiFormRow } from '@elastic/eui';
|
||||||
import { i18n } from '@kbn/i18n';
|
import { i18n } from '@kbn/i18n';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React, { useCallback } from 'react';
|
import React, { useCallback } from 'react';
|
||||||
|
import { QualityWarning } from '../../../../../common/log_analysis';
|
||||||
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
|
import { LoadingOverlayWrapper } from '../../../loading_overlay_wrapper';
|
||||||
import { IndexSetupRow } from './index_setup_row';
|
import { IndexSetupRow } from './index_setup_row';
|
||||||
import { AvailableIndex, ValidationIndicesError } from './validation';
|
import { AvailableIndex, ValidationIndicesError } from './validation';
|
||||||
|
@ -17,12 +18,14 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
||||||
indices: AvailableIndex[];
|
indices: AvailableIndex[];
|
||||||
isValidating: boolean;
|
isValidating: boolean;
|
||||||
onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
|
onChangeSelectedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||||
|
previousQualityWarnings?: QualityWarning[];
|
||||||
validationErrors?: ValidationIndicesError[];
|
validationErrors?: ValidationIndicesError[];
|
||||||
}> = ({
|
}> = ({
|
||||||
disabled = false,
|
disabled = false,
|
||||||
indices,
|
indices,
|
||||||
isValidating,
|
isValidating,
|
||||||
onChangeSelectedIndices,
|
onChangeSelectedIndices,
|
||||||
|
previousQualityWarnings = [],
|
||||||
validationErrors = [],
|
validationErrors = [],
|
||||||
}) => {
|
}) => {
|
||||||
const changeIsIndexSelected = useCallback(
|
const changeIsIndexSelected = useCallback(
|
||||||
|
@ -81,6 +84,7 @@ export const AnalysisSetupIndicesForm: React.FunctionComponent<{
|
||||||
key={index.name}
|
key={index.name}
|
||||||
onChangeIsSelected={changeIsIndexSelected}
|
onChangeIsSelected={changeIsIndexSelected}
|
||||||
onChangeDatasetFilter={changeDatasetFilter}
|
onChangeDatasetFilter={changeDatasetFilter}
|
||||||
|
previousQualityWarnings={previousQualityWarnings}
|
||||||
/>
|
/>
|
||||||
))}
|
))}
|
||||||
</>
|
</>
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
import {
|
import {
|
||||||
EuiFilterButton,
|
EuiFilterButton,
|
||||||
EuiFilterGroup,
|
EuiFilterGroup,
|
||||||
|
EuiIconTip,
|
||||||
EuiPopover,
|
EuiPopover,
|
||||||
EuiPopoverTitle,
|
EuiPopoverTitle,
|
||||||
EuiSelectable,
|
EuiSelectable,
|
||||||
|
@ -14,11 +15,15 @@ import {
|
||||||
} from '@elastic/eui';
|
} from '@elastic/eui';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React, { useCallback, useMemo } from 'react';
|
import React, { useCallback, useMemo } from 'react';
|
||||||
import { DatasetFilter } from '../../../../../common/log_analysis';
|
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
|
||||||
import { useVisibilityState } from '../../../../utils/use_visibility_state';
|
import { useVisibilityState } from '../../../../utils/use_visibility_state';
|
||||||
|
import { CategoryQualityWarningReasonDescription } from '../../log_analysis_job_status/quality_warning_notices';
|
||||||
|
|
||||||
export const IndexSetupDatasetFilter: React.FC<{
|
export const IndexSetupDatasetFilter: React.FC<{
|
||||||
availableDatasets: string[];
|
availableDatasets: Array<{
|
||||||
|
dataset: string;
|
||||||
|
warnings: QualityWarning[];
|
||||||
|
}>;
|
||||||
datasetFilter: DatasetFilter;
|
datasetFilter: DatasetFilter;
|
||||||
isDisabled?: boolean;
|
isDisabled?: boolean;
|
||||||
onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
|
onChangeDatasetFilter: (datasetFilter: DatasetFilter) => void;
|
||||||
|
@ -40,12 +45,13 @@ export const IndexSetupDatasetFilter: React.FC<{
|
||||||
[onChangeDatasetFilter]
|
[onChangeDatasetFilter]
|
||||||
);
|
);
|
||||||
|
|
||||||
const selectableOptions: EuiSelectableOption[] = useMemo(
|
const selectableOptions = useMemo<EuiSelectableOption[]>(
|
||||||
() =>
|
() =>
|
||||||
availableDatasets.map((datasetName) => ({
|
availableDatasets.map(({ dataset, warnings }) => ({
|
||||||
label: datasetName,
|
label: dataset,
|
||||||
|
append: warnings.length > 0 ? <DatasetWarningMarker warnings={warnings} /> : null,
|
||||||
checked:
|
checked:
|
||||||
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(datasetName)
|
datasetFilter.type === 'includeSome' && datasetFilter.datasets.includes(dataset)
|
||||||
? 'on'
|
? 'on'
|
||||||
: undefined,
|
: undefined,
|
||||||
})),
|
})),
|
||||||
|
@ -86,3 +92,15 @@ export const IndexSetupDatasetFilter: React.FC<{
|
||||||
</EuiFilterGroup>
|
</EuiFilterGroup>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const DatasetWarningMarker: React.FC<{ warnings: QualityWarning[] }> = ({ warnings }) => {
|
||||||
|
const warningDescriptions = warnings.flatMap((warning) =>
|
||||||
|
warning.type === 'categoryQualityWarning'
|
||||||
|
? warning.reasons.map((reason) => (
|
||||||
|
<CategoryQualityWarningReasonDescription key={reason.type} reason={reason} />
|
||||||
|
))
|
||||||
|
: []
|
||||||
|
);
|
||||||
|
|
||||||
|
return <EuiIconTip content={warningDescriptions} type="alert" color="warning" />;
|
||||||
|
};
|
||||||
|
|
|
@ -4,10 +4,10 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIcon, EuiToolTip } from '@elastic/eui';
|
import { EuiCheckbox, EuiCode, EuiFlexGroup, EuiFlexItem, EuiIconTip } from '@elastic/eui';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React, { useCallback } from 'react';
|
import React, { useCallback, useMemo } from 'react';
|
||||||
import { DatasetFilter } from '../../../../../common/log_analysis';
|
import { DatasetFilter, QualityWarning } from '../../../../../common/log_analysis';
|
||||||
import { IndexSetupDatasetFilter } from './index_setup_dataset_filter';
|
import { IndexSetupDatasetFilter } from './index_setup_dataset_filter';
|
||||||
import { AvailableIndex, ValidationUIError } from './validation';
|
import { AvailableIndex, ValidationUIError } from './validation';
|
||||||
|
|
||||||
|
@ -16,7 +16,14 @@ export const IndexSetupRow: React.FC<{
|
||||||
isDisabled: boolean;
|
isDisabled: boolean;
|
||||||
onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void;
|
onChangeDatasetFilter: (indexName: string, datasetFilter: DatasetFilter) => void;
|
||||||
onChangeIsSelected: (indexName: string, isSelected: boolean) => void;
|
onChangeIsSelected: (indexName: string, isSelected: boolean) => void;
|
||||||
}> = ({ index, isDisabled, onChangeDatasetFilter, onChangeIsSelected }) => {
|
previousQualityWarnings: QualityWarning[];
|
||||||
|
}> = ({
|
||||||
|
index,
|
||||||
|
isDisabled,
|
||||||
|
onChangeDatasetFilter,
|
||||||
|
onChangeIsSelected,
|
||||||
|
previousQualityWarnings,
|
||||||
|
}) => {
|
||||||
const changeIsSelected = useCallback(
|
const changeIsSelected = useCallback(
|
||||||
(event: React.ChangeEvent<HTMLInputElement>) => {
|
(event: React.ChangeEvent<HTMLInputElement>) => {
|
||||||
onChangeIsSelected(index.name, event.currentTarget.checked);
|
onChangeIsSelected(index.name, event.currentTarget.checked);
|
||||||
|
@ -29,6 +36,29 @@ export const IndexSetupRow: React.FC<{
|
||||||
[index.name, onChangeDatasetFilter]
|
[index.name, onChangeDatasetFilter]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
const datasets = useMemo(
|
||||||
|
() =>
|
||||||
|
index.validity === 'valid'
|
||||||
|
? index.availableDatasets.map((availableDataset) => ({
|
||||||
|
dataset: availableDataset,
|
||||||
|
warnings: previousQualityWarnings.filter(({ dataset }) => dataset === availableDataset),
|
||||||
|
}))
|
||||||
|
: [],
|
||||||
|
[index, previousQualityWarnings]
|
||||||
|
);
|
||||||
|
|
||||||
|
const datasetIndependentQualityWarnings = useMemo(
|
||||||
|
() => previousQualityWarnings.filter(({ dataset }) => dataset === ''),
|
||||||
|
[previousQualityWarnings]
|
||||||
|
);
|
||||||
|
|
||||||
|
const hasWarnings = useMemo(
|
||||||
|
() =>
|
||||||
|
datasetIndependentQualityWarnings.length > 0 ||
|
||||||
|
datasets.some(({ warnings }) => warnings.length > 0),
|
||||||
|
[datasetIndependentQualityWarnings, datasets]
|
||||||
|
);
|
||||||
|
|
||||||
const isSelected = index.validity === 'valid' && index.isSelected;
|
const isSelected = index.validity === 'valid' && index.isSelected;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
@ -37,7 +67,23 @@ export const IndexSetupRow: React.FC<{
|
||||||
<EuiCheckbox
|
<EuiCheckbox
|
||||||
key={index.name}
|
key={index.name}
|
||||||
id={index.name}
|
id={index.name}
|
||||||
label={<EuiCode>{index.name}</EuiCode>}
|
label={
|
||||||
|
<>
|
||||||
|
<EuiCode>{index.name}</EuiCode>{' '}
|
||||||
|
{index.validity === 'valid' && hasWarnings ? (
|
||||||
|
<EuiIconTip
|
||||||
|
content={
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.logs.analsysisSetup.indexQualityWarningTooltipMessage"
|
||||||
|
defaultMessage="While analyzing the log messages from these indices we've detected some problems which might indicate a reduced quality of the results. Consider excluding these indices or problematic datasets from the analysis."
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
type="alert"
|
||||||
|
color="warning"
|
||||||
|
/>
|
||||||
|
) : null}
|
||||||
|
</>
|
||||||
|
}
|
||||||
onChange={changeIsSelected}
|
onChange={changeIsSelected}
|
||||||
checked={isSelected}
|
checked={isSelected}
|
||||||
disabled={isDisabled || index.validity === 'invalid'}
|
disabled={isDisabled || index.validity === 'invalid'}
|
||||||
|
@ -45,12 +91,10 @@ export const IndexSetupRow: React.FC<{
|
||||||
</EuiFlexItem>
|
</EuiFlexItem>
|
||||||
<EuiFlexItem grow={false}>
|
<EuiFlexItem grow={false}>
|
||||||
{index.validity === 'invalid' ? (
|
{index.validity === 'invalid' ? (
|
||||||
<EuiToolTip content={formatValidationError(index.errors)}>
|
<EuiIconTip content={formatValidationError(index.errors)} type="alert" color="danger" />
|
||||||
<EuiIcon type="alert" color="danger" />
|
|
||||||
</EuiToolTip>
|
|
||||||
) : index.validity === 'valid' ? (
|
) : index.validity === 'valid' ? (
|
||||||
<IndexSetupDatasetFilter
|
<IndexSetupDatasetFilter
|
||||||
availableDatasets={index.availableDatasets}
|
availableDatasets={datasets}
|
||||||
datasetFilter={index.datasetFilter}
|
datasetFilter={index.datasetFilter}
|
||||||
isDisabled={!isSelected || isDisabled}
|
isDisabled={!isSelected || isDisabled}
|
||||||
onChangeDatasetFilter={changeDatasetFilter}
|
onChangeDatasetFilter={changeDatasetFilter}
|
||||||
|
|
|
@ -0,0 +1,104 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { actions } from '@storybook/addon-actions';
|
||||||
|
import { storiesOf } from '@storybook/react';
|
||||||
|
import React from 'react';
|
||||||
|
import { EuiThemeProvider } from '../../../../../../observability/public';
|
||||||
|
import { InitialConfigurationStep } from './initial_configuration_step';
|
||||||
|
|
||||||
|
storiesOf('infra/logAnalysis/SetupInitialConfigurationStep', module)
|
||||||
|
.addDecorator((renderStory) => (
|
||||||
|
<EuiThemeProvider>
|
||||||
|
<div style={{ maxWidth: 800 }}>{renderStory()}</div>
|
||||||
|
</EuiThemeProvider>
|
||||||
|
))
|
||||||
|
.add('Reconfiguration with partitioned warnings', () => {
|
||||||
|
return (
|
||||||
|
<InitialConfigurationStep
|
||||||
|
{...storyActions}
|
||||||
|
startTime={Date.now()}
|
||||||
|
endTime={undefined}
|
||||||
|
isValidating={false}
|
||||||
|
setupStatus={{ type: 'required' }}
|
||||||
|
validatedIndices={[
|
||||||
|
{
|
||||||
|
name: 'index-1-*',
|
||||||
|
validity: 'valid',
|
||||||
|
isSelected: true,
|
||||||
|
datasetFilter: { type: 'includeAll' },
|
||||||
|
availableDatasets: ['first', 'second', 'third'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'index-2-*',
|
||||||
|
validity: 'invalid',
|
||||||
|
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
previousQualityWarnings={[
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'job-1',
|
||||||
|
dataset: 'second',
|
||||||
|
reasons: [
|
||||||
|
{ type: 'noFrequentCategories' },
|
||||||
|
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'job-1',
|
||||||
|
dataset: 'third',
|
||||||
|
reasons: [{ type: 'singleCategory' }],
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.add('Reconfiguration with unpartitioned warnings', () => {
|
||||||
|
return (
|
||||||
|
<InitialConfigurationStep
|
||||||
|
{...storyActions}
|
||||||
|
startTime={Date.now()}
|
||||||
|
endTime={undefined}
|
||||||
|
isValidating={false}
|
||||||
|
setupStatus={{ type: 'required' }}
|
||||||
|
validatedIndices={[
|
||||||
|
{
|
||||||
|
name: 'index-1-*',
|
||||||
|
validity: 'valid',
|
||||||
|
isSelected: true,
|
||||||
|
datasetFilter: { type: 'includeAll' },
|
||||||
|
availableDatasets: ['first', 'second', 'third'],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: 'index-2-*',
|
||||||
|
validity: 'invalid',
|
||||||
|
errors: [{ index: 'index-2-*', error: 'INDEX_NOT_FOUND' }],
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
previousQualityWarnings={[
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'job-1',
|
||||||
|
dataset: '',
|
||||||
|
reasons: [
|
||||||
|
{ type: 'noFrequentCategories' },
|
||||||
|
{ type: 'manyDeadCategories', deadCategoriesRatio: 0.9 },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'categoryQualityWarning',
|
||||||
|
jobId: 'job-1',
|
||||||
|
dataset: '',
|
||||||
|
reasons: [{ type: 'singleCategory' }],
|
||||||
|
},
|
||||||
|
]}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const storyActions = actions('setStartTime', 'setEndTime', 'setValidatedIndices');
|
|
@ -9,7 +9,7 @@ import { EuiContainedStepProps } from '@elastic/eui/src/components/steps/steps';
|
||||||
import { i18n } from '@kbn/i18n';
|
import { i18n } from '@kbn/i18n';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React, { useMemo } from 'react';
|
import React, { useMemo } from 'react';
|
||||||
import { SetupStatus } from '../../../../../common/log_analysis';
|
import { QualityWarning, SetupStatus } from '../../../../../common/log_analysis';
|
||||||
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
|
import { AnalysisSetupIndicesForm } from './analysis_setup_indices_form';
|
||||||
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
|
import { AnalysisSetupTimerangeForm } from './analysis_setup_timerange_form';
|
||||||
import {
|
import {
|
||||||
|
@ -31,6 +31,7 @@ interface InitialConfigurationStepProps {
|
||||||
setupStatus: SetupStatus;
|
setupStatus: SetupStatus;
|
||||||
setValidatedIndices: (selectedIndices: AvailableIndex[]) => void;
|
setValidatedIndices: (selectedIndices: AvailableIndex[]) => void;
|
||||||
validationErrors?: ValidationUIError[];
|
validationErrors?: ValidationUIError[];
|
||||||
|
previousQualityWarnings?: QualityWarning[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export const createInitialConfigurationStep = (
|
export const createInitialConfigurationStep = (
|
||||||
|
@ -50,6 +51,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
|
||||||
setupStatus,
|
setupStatus,
|
||||||
setValidatedIndices,
|
setValidatedIndices,
|
||||||
validationErrors = [],
|
validationErrors = [],
|
||||||
|
previousQualityWarnings = [],
|
||||||
}: InitialConfigurationStepProps) => {
|
}: InitialConfigurationStepProps) => {
|
||||||
const disabled = useMemo(() => !editableFormStatus.includes(setupStatus.type), [setupStatus]);
|
const disabled = useMemo(() => !editableFormStatus.includes(setupStatus.type), [setupStatus]);
|
||||||
|
|
||||||
|
@ -75,6 +77,7 @@ export const InitialConfigurationStep: React.FunctionComponent<InitialConfigurat
|
||||||
indices={validatedIndices}
|
indices={validatedIndices}
|
||||||
isValidating={isValidating}
|
isValidating={isValidating}
|
||||||
onChangeSelectedIndices={setValidatedIndices}
|
onChangeSelectedIndices={setValidatedIndices}
|
||||||
|
previousQualityWarnings={previousQualityWarnings}
|
||||||
validationErrors={indexValidationErrors}
|
validationErrors={indexValidationErrors}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
|
|
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
import { EuiSpacer, EuiSteps, EuiText, EuiTitle } from '@elastic/eui';
|
import { EuiSpacer, EuiSteps, EuiText, EuiTitle } from '@elastic/eui';
|
||||||
import React, { useCallback, useMemo } from 'react';
|
import React, { useCallback, useMemo } from 'react';
|
||||||
|
import { useMount } from 'react-use';
|
||||||
import { useLogEntryCategoriesSetup } from '../../../../containers/logs/log_analysis/modules/log_entry_categories';
|
import { useLogEntryCategoriesSetup } from '../../../../containers/logs/log_analysis/modules/log_entry_categories';
|
||||||
import { createInitialConfigurationStep } from '../initial_configuration_step';
|
import { createInitialConfigurationStep } from '../initial_configuration_step';
|
||||||
import { createProcessStep } from '../process_step';
|
import { createProcessStep } from '../process_step';
|
||||||
|
@ -14,8 +15,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
||||||
onClose: () => void;
|
onClose: () => void;
|
||||||
}> = ({ onClose }) => {
|
}> = ({ onClose }) => {
|
||||||
const {
|
const {
|
||||||
|
categoryQualityWarnings,
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
endTime,
|
endTime,
|
||||||
|
fetchJobStatus,
|
||||||
isValidating,
|
isValidating,
|
||||||
lastSetupErrorMessages,
|
lastSetupErrorMessages,
|
||||||
moduleDescriptor,
|
moduleDescriptor,
|
||||||
|
@ -30,6 +33,10 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
||||||
viewResults,
|
viewResults,
|
||||||
} = useLogEntryCategoriesSetup();
|
} = useLogEntryCategoriesSetup();
|
||||||
|
|
||||||
|
useMount(() => {
|
||||||
|
fetchJobStatus();
|
||||||
|
});
|
||||||
|
|
||||||
const viewResultsAndClose = useCallback(() => {
|
const viewResultsAndClose = useCallback(() => {
|
||||||
viewResults();
|
viewResults();
|
||||||
onClose();
|
onClose();
|
||||||
|
@ -47,6 +54,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
||||||
setupStatus,
|
setupStatus,
|
||||||
setValidatedIndices,
|
setValidatedIndices,
|
||||||
validationErrors,
|
validationErrors,
|
||||||
|
previousQualityWarnings: categoryQualityWarnings,
|
||||||
}),
|
}),
|
||||||
createProcessStep({
|
createProcessStep({
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
|
@ -58,6 +66,7 @@ export const LogEntryCategoriesSetupView: React.FC<{
|
||||||
}),
|
}),
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
|
categoryQualityWarnings,
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
endTime,
|
endTime,
|
||||||
isValidating,
|
isValidating,
|
||||||
|
|
|
@ -15,14 +15,16 @@ import {
|
||||||
} from '@elastic/eui';
|
} from '@elastic/eui';
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
import { FormattedMessage } from '@kbn/i18n/react';
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
|
|
||||||
import { LogEntryCategoriesSetupView } from './log_entry_categories_setup_view';
|
import { LogEntryCategoriesSetupView } from './log_entry_categories_setup_view';
|
||||||
|
import { LogEntryRateSetupView } from './log_entry_rate_setup_view';
|
||||||
import { LogAnalysisModuleList } from './module_list';
|
import { LogAnalysisModuleList } from './module_list';
|
||||||
import { useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
|
import { ModuleId, moduleIds, useLogAnalysisSetupFlyoutStateContext } from './setup_flyout_state';
|
||||||
|
|
||||||
const FLYOUT_HEADING_ID = 'logAnalysisSetupFlyoutHeading';
|
const FLYOUT_HEADING_ID = 'logAnalysisSetupFlyoutHeading';
|
||||||
|
|
||||||
export const LogAnalysisSetupFlyout: React.FC = () => {
|
export const LogAnalysisSetupFlyout: React.FC<{
|
||||||
|
allowedModules?: ModuleId[];
|
||||||
|
}> = ({ allowedModules = moduleIds }) => {
|
||||||
const {
|
const {
|
||||||
closeFlyout,
|
closeFlyout,
|
||||||
flyoutView,
|
flyoutView,
|
||||||
|
@ -49,32 +51,58 @@ export const LogAnalysisSetupFlyout: React.FC = () => {
|
||||||
<EuiFlyoutBody>
|
<EuiFlyoutBody>
|
||||||
{flyoutView.view === 'moduleList' ? (
|
{flyoutView.view === 'moduleList' ? (
|
||||||
<LogAnalysisModuleList onViewModuleSetup={showModuleSetup} />
|
<LogAnalysisModuleList onViewModuleSetup={showModuleSetup} />
|
||||||
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_analysis' ? (
|
) : flyoutView.view === 'moduleSetup' && allowedModules.includes(flyoutView.module) ? (
|
||||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
|
<ModuleSetupView
|
||||||
<LogEntryRateSetupView onClose={closeFlyout} />
|
moduleId={flyoutView.module}
|
||||||
</LogAnalysisSetupFlyoutSubPage>
|
onClose={closeFlyout}
|
||||||
) : flyoutView.view === 'moduleSetup' && flyoutView.module === 'logs_ui_categories' ? (
|
onViewModuleList={allowedModules.length > 1 ? showModuleList : undefined}
|
||||||
<LogAnalysisSetupFlyoutSubPage onViewModuleList={showModuleList}>
|
/>
|
||||||
<LogEntryCategoriesSetupView onClose={closeFlyout} />
|
|
||||||
</LogAnalysisSetupFlyoutSubPage>
|
|
||||||
) : null}
|
) : null}
|
||||||
</EuiFlyoutBody>
|
</EuiFlyoutBody>
|
||||||
</EuiFlyout>
|
</EuiFlyout>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const ModuleSetupView: React.FC<{
|
||||||
|
moduleId: ModuleId;
|
||||||
|
onClose: () => void;
|
||||||
|
onViewModuleList?: () => void;
|
||||||
|
}> = ({ moduleId, onClose, onViewModuleList }) => {
|
||||||
|
switch (moduleId) {
|
||||||
|
case 'logs_ui_analysis':
|
||||||
|
return (
|
||||||
|
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
|
||||||
|
<LogEntryRateSetupView onClose={onClose} />
|
||||||
|
</LogAnalysisSetupFlyoutSubPage>
|
||||||
|
);
|
||||||
|
case 'logs_ui_categories':
|
||||||
|
return (
|
||||||
|
<LogAnalysisSetupFlyoutSubPage onViewModuleList={onViewModuleList}>
|
||||||
|
<LogEntryCategoriesSetupView onClose={onClose} />
|
||||||
|
</LogAnalysisSetupFlyoutSubPage>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const LogAnalysisSetupFlyoutSubPage: React.FC<{
|
const LogAnalysisSetupFlyoutSubPage: React.FC<{
|
||||||
onViewModuleList: () => void;
|
onViewModuleList?: () => void;
|
||||||
}> = ({ children, onViewModuleList }) => (
|
}> = ({ children, onViewModuleList }) => (
|
||||||
<EuiFlexGroup alignItems="flexStart" direction="column" gutterSize="none">
|
<EuiFlexGroup alignItems="flexStart" direction="column" gutterSize="none">
|
||||||
<EuiFlexItem grow={false}>
|
{onViewModuleList ? (
|
||||||
<EuiButtonEmpty flush="left" iconSide="left" iconType="arrowLeft" onClick={onViewModuleList}>
|
<EuiFlexItem grow={false}>
|
||||||
<FormattedMessage
|
<EuiButtonEmpty
|
||||||
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
|
flush="left"
|
||||||
defaultMessage="All Machine Learning jobs"
|
iconSide="left"
|
||||||
/>
|
iconType="arrowLeft"
|
||||||
</EuiButtonEmpty>
|
onClick={onViewModuleList}
|
||||||
</EuiFlexItem>
|
>
|
||||||
|
<FormattedMessage
|
||||||
|
id="xpack.infra.logs.analysis.setupFlyoutGotoListButtonLabel"
|
||||||
|
defaultMessage="All Machine Learning jobs"
|
||||||
|
/>
|
||||||
|
</EuiButtonEmpty>
|
||||||
|
</EuiFlexItem>
|
||||||
|
) : null}
|
||||||
<EuiFlexItem>{children}</EuiFlexItem>
|
<EuiFlexItem>{children}</EuiFlexItem>
|
||||||
</EuiFlexGroup>
|
</EuiFlexGroup>
|
||||||
);
|
);
|
||||||
|
|
|
@ -9,6 +9,8 @@ import { useState, useCallback } from 'react';
|
||||||
|
|
||||||
export type ModuleId = 'logs_ui_analysis' | 'logs_ui_categories';
|
export type ModuleId = 'logs_ui_analysis' | 'logs_ui_categories';
|
||||||
|
|
||||||
|
export const moduleIds = ['logs_ui_analysis', 'logs_ui_categories'] as const;
|
||||||
|
|
||||||
type FlyoutView =
|
type FlyoutView =
|
||||||
| { view: 'hidden' }
|
| { view: 'hidden' }
|
||||||
| { view: 'moduleList' }
|
| { view: 'moduleList' }
|
||||||
|
|
|
@ -0,0 +1,47 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { HttpHandler } from 'src/core/public';
|
||||||
|
import {
|
||||||
|
CategorizerStatus,
|
||||||
|
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
|
||||||
|
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
|
||||||
|
LogEntryCategoriesDatasetStats,
|
||||||
|
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||||
|
} from '../../../../../common/http_api';
|
||||||
|
import { decodeOrThrow } from '../../../../../common/runtime_types';
|
||||||
|
|
||||||
|
export { LogEntryCategoriesDatasetStats };
|
||||||
|
|
||||||
|
export const callGetLatestCategoriesDatasetsStatsAPI = async (
|
||||||
|
{
|
||||||
|
jobIds,
|
||||||
|
startTime,
|
||||||
|
endTime,
|
||||||
|
includeCategorizerStatuses,
|
||||||
|
}: {
|
||||||
|
jobIds: string[];
|
||||||
|
startTime: number;
|
||||||
|
endTime: number;
|
||||||
|
includeCategorizerStatuses: CategorizerStatus[];
|
||||||
|
},
|
||||||
|
fetch: HttpHandler
|
||||||
|
) => {
|
||||||
|
const response = await fetch(LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH, {
|
||||||
|
method: 'POST',
|
||||||
|
body: JSON.stringify(
|
||||||
|
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT.encode({
|
||||||
|
data: {
|
||||||
|
jobIds,
|
||||||
|
timeRange: { startTime, endTime },
|
||||||
|
includeCategorizerStatuses,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
),
|
||||||
|
});
|
||||||
|
|
||||||
|
return decodeOrThrow(getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT)(response);
|
||||||
|
};
|
|
@ -54,6 +54,17 @@ const jobStateRT = rt.keyof({
|
||||||
opening: null,
|
opening: null,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const jobAnalysisConfigRT = rt.partial({
|
||||||
|
per_partition_categorization: rt.intersection([
|
||||||
|
rt.type({
|
||||||
|
enabled: rt.boolean,
|
||||||
|
}),
|
||||||
|
rt.partial({
|
||||||
|
stop_on_warn: rt.boolean,
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
});
|
||||||
|
|
||||||
const jobCategorizationStatusRT = rt.keyof({
|
const jobCategorizationStatusRT = rt.keyof({
|
||||||
ok: null,
|
ok: null,
|
||||||
warn: null,
|
warn: null,
|
||||||
|
@ -64,6 +75,7 @@ const jobModelSizeStatsRT = rt.type({
|
||||||
categorized_doc_count: rt.number,
|
categorized_doc_count: rt.number,
|
||||||
dead_category_count: rt.number,
|
dead_category_count: rt.number,
|
||||||
frequent_category_count: rt.number,
|
frequent_category_count: rt.number,
|
||||||
|
log_time: rt.number,
|
||||||
rare_category_count: rt.number,
|
rare_category_count: rt.number,
|
||||||
total_category_count: rt.number,
|
total_category_count: rt.number,
|
||||||
});
|
});
|
||||||
|
@ -79,6 +91,8 @@ export const jobSummaryRT = rt.intersection([
|
||||||
datafeedIndices: rt.array(rt.string),
|
datafeedIndices: rt.array(rt.string),
|
||||||
datafeedState: datafeedStateRT,
|
datafeedState: datafeedStateRT,
|
||||||
fullJob: rt.partial({
|
fullJob: rt.partial({
|
||||||
|
analysis_config: jobAnalysisConfigRT,
|
||||||
|
create_time: rt.number,
|
||||||
custom_settings: jobCustomSettingsRT,
|
custom_settings: jobCustomSettingsRT,
|
||||||
finished_time: rt.number,
|
finished_time: rt.number,
|
||||||
model_size_stats: jobModelSizeStatsRT,
|
model_size_stats: jobModelSizeStatsRT,
|
||||||
|
|
|
@ -50,43 +50,3 @@ export interface ModuleSourceConfiguration {
|
||||||
spaceId: string;
|
spaceId: string;
|
||||||
timestampField: string;
|
timestampField: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface ManyCategoriesWarningReason {
|
|
||||||
type: 'manyCategories';
|
|
||||||
categoriesDocumentRatio: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ManyDeadCategoriesWarningReason {
|
|
||||||
type: 'manyDeadCategories';
|
|
||||||
deadCategoriesRatio: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface ManyRareCategoriesWarningReason {
|
|
||||||
type: 'manyRareCategories';
|
|
||||||
rareCategoriesRatio: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface NoFrequentCategoriesWarningReason {
|
|
||||||
type: 'noFrequentCategories';
|
|
||||||
}
|
|
||||||
|
|
||||||
interface SingleCategoryWarningReason {
|
|
||||||
type: 'singleCategory';
|
|
||||||
}
|
|
||||||
|
|
||||||
export type CategoryQualityWarningReason =
|
|
||||||
| ManyCategoriesWarningReason
|
|
||||||
| ManyDeadCategoriesWarningReason
|
|
||||||
| ManyRareCategoriesWarningReason
|
|
||||||
| NoFrequentCategoriesWarningReason
|
|
||||||
| SingleCategoryWarningReason;
|
|
||||||
|
|
||||||
export type CategoryQualityWarningReasonType = CategoryQualityWarningReason['type'];
|
|
||||||
|
|
||||||
export interface CategoryQualityWarning {
|
|
||||||
type: 'categoryQualityWarning';
|
|
||||||
jobId: string;
|
|
||||||
reasons: CategoryQualityWarningReason[];
|
|
||||||
}
|
|
||||||
|
|
||||||
export type QualityWarning = CategoryQualityWarning;
|
|
||||||
|
|
|
@ -4,43 +4,124 @@
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { useMemo } from 'react';
|
import { useMemo, useState } from 'react';
|
||||||
|
import { useDeepCompareEffect } from 'react-use';
|
||||||
import {
|
import {
|
||||||
JobModelSizeStats,
|
|
||||||
JobSummary,
|
|
||||||
QualityWarning,
|
|
||||||
CategoryQualityWarningReason,
|
CategoryQualityWarningReason,
|
||||||
} from '../../log_analysis_module_types';
|
QualityWarning,
|
||||||
|
} from '../../../../../../common/log_analysis';
|
||||||
|
import { useKibanaContextForPlugin } from '../../../../../hooks/use_kibana';
|
||||||
|
import { useTrackedPromise } from '../../../../../utils/use_tracked_promise';
|
||||||
|
import {
|
||||||
|
callGetLatestCategoriesDatasetsStatsAPI,
|
||||||
|
LogEntryCategoriesDatasetStats,
|
||||||
|
} from '../../api/get_latest_categories_datasets_stats';
|
||||||
|
import { JobModelSizeStats, JobSummary } from '../../log_analysis_module_types';
|
||||||
|
|
||||||
export const useLogEntryCategoriesQuality = ({ jobSummaries }: { jobSummaries: JobSummary[] }) => {
|
export const useLogEntryCategoriesQuality = ({ jobSummaries }: { jobSummaries: JobSummary[] }) => {
|
||||||
|
const {
|
||||||
|
services: {
|
||||||
|
http: { fetch },
|
||||||
|
},
|
||||||
|
} = useKibanaContextForPlugin();
|
||||||
|
|
||||||
|
const [lastestWarnedDatasetsStats, setLatestWarnedDatasetsStats] = useState<
|
||||||
|
LogEntryCategoriesDatasetStats[]
|
||||||
|
>([]);
|
||||||
|
|
||||||
|
const jobSummariesWithCategoryWarnings = useMemo(
|
||||||
|
() => jobSummaries.filter(isJobWithCategoryWarnings),
|
||||||
|
[jobSummaries]
|
||||||
|
);
|
||||||
|
|
||||||
|
const jobSummariesWithPartitionedCategoryWarnings = useMemo(
|
||||||
|
() => jobSummariesWithCategoryWarnings.filter(isJobWithPartitionedCategories),
|
||||||
|
[jobSummariesWithCategoryWarnings]
|
||||||
|
);
|
||||||
|
|
||||||
|
const [fetchLatestWarnedDatasetsStatsRequest, fetchLatestWarnedDatasetsStats] = useTrackedPromise(
|
||||||
|
{
|
||||||
|
cancelPreviousOn: 'creation',
|
||||||
|
createPromise: (
|
||||||
|
statsIntervals: Array<{ jobId: string; startTime: number; endTime: number }>
|
||||||
|
) =>
|
||||||
|
Promise.all(
|
||||||
|
statsIntervals.map(({ jobId, startTime, endTime }) =>
|
||||||
|
callGetLatestCategoriesDatasetsStatsAPI(
|
||||||
|
{ jobIds: [jobId], startTime, endTime, includeCategorizerStatuses: ['warn'] },
|
||||||
|
fetch
|
||||||
|
)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
onResolve: (results) => {
|
||||||
|
setLatestWarnedDatasetsStats(results.flatMap(({ data: { datasetStats } }) => datasetStats));
|
||||||
|
},
|
||||||
|
},
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
|
useDeepCompareEffect(() => {
|
||||||
|
fetchLatestWarnedDatasetsStats(
|
||||||
|
jobSummariesWithPartitionedCategoryWarnings.map((jobSummary) => ({
|
||||||
|
jobId: jobSummary.id,
|
||||||
|
startTime: jobSummary.fullJob?.create_time ?? 0,
|
||||||
|
endTime: jobSummary.fullJob?.model_size_stats?.log_time ?? Date.now(),
|
||||||
|
}))
|
||||||
|
);
|
||||||
|
}, [jobSummariesWithPartitionedCategoryWarnings]);
|
||||||
|
|
||||||
const categoryQualityWarnings: QualityWarning[] = useMemo(
|
const categoryQualityWarnings: QualityWarning[] = useMemo(
|
||||||
() =>
|
() => [
|
||||||
jobSummaries
|
...jobSummariesWithCategoryWarnings
|
||||||
.filter(
|
.filter((jobSummary) => !isJobWithPartitionedCategories(jobSummary))
|
||||||
(jobSummary) => jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn'
|
|
||||||
)
|
|
||||||
.map((jobSummary) => ({
|
.map((jobSummary) => ({
|
||||||
type: 'categoryQualityWarning',
|
type: 'categoryQualityWarning' as const,
|
||||||
jobId: jobSummary.id,
|
jobId: jobSummary.id,
|
||||||
|
dataset: '',
|
||||||
reasons: jobSummary.fullJob?.model_size_stats
|
reasons: jobSummary.fullJob?.model_size_stats
|
||||||
? getCategoryQualityWarningReasons(jobSummary.fullJob.model_size_stats)
|
? getCategoryQualityWarningReasons(jobSummary.fullJob.model_size_stats)
|
||||||
: [],
|
: [],
|
||||||
})),
|
})),
|
||||||
[jobSummaries]
|
...lastestWarnedDatasetsStats.map((datasetStats) => ({
|
||||||
|
type: 'categoryQualityWarning' as const,
|
||||||
|
jobId: datasetStats.job_id,
|
||||||
|
dataset: datasetStats.dataset,
|
||||||
|
reasons: getCategoryQualityWarningReasons(datasetStats),
|
||||||
|
})),
|
||||||
|
],
|
||||||
|
[jobSummariesWithCategoryWarnings, lastestWarnedDatasetsStats]
|
||||||
);
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
categoryQualityWarnings,
|
categoryQualityWarnings,
|
||||||
|
lastLatestWarnedDatasetsStatsRequestErrors:
|
||||||
|
fetchLatestWarnedDatasetsStatsRequest.state === 'rejected'
|
||||||
|
? fetchLatestWarnedDatasetsStatsRequest.value
|
||||||
|
: null,
|
||||||
|
isLoadingCategoryQualityWarnings: fetchLatestWarnedDatasetsStatsRequest.state === 'pending',
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const isJobWithCategoryWarnings = (jobSummary: JobSummary) =>
|
||||||
|
jobSummary.fullJob?.model_size_stats?.categorization_status === 'warn';
|
||||||
|
|
||||||
|
const isJobWithPartitionedCategories = (jobSummary: JobSummary) =>
|
||||||
|
jobSummary.fullJob?.analysis_config?.per_partition_categorization ?? false;
|
||||||
|
|
||||||
const getCategoryQualityWarningReasons = ({
|
const getCategoryQualityWarningReasons = ({
|
||||||
categorized_doc_count: categorizedDocCount,
|
categorized_doc_count: categorizedDocCount,
|
||||||
dead_category_count: deadCategoryCount,
|
dead_category_count: deadCategoryCount,
|
||||||
frequent_category_count: frequentCategoryCount,
|
frequent_category_count: frequentCategoryCount,
|
||||||
rare_category_count: rareCategoryCount,
|
rare_category_count: rareCategoryCount,
|
||||||
total_category_count: totalCategoryCount,
|
total_category_count: totalCategoryCount,
|
||||||
}: JobModelSizeStats): CategoryQualityWarningReason[] => {
|
}: Pick<
|
||||||
|
JobModelSizeStats,
|
||||||
|
| 'categorized_doc_count'
|
||||||
|
| 'dead_category_count'
|
||||||
|
| 'frequent_category_count'
|
||||||
|
| 'rare_category_count'
|
||||||
|
| 'total_category_count'
|
||||||
|
>): CategoryQualityWarningReason[] => {
|
||||||
const rareCategoriesRatio = rareCategoryCount / totalCategoryCount;
|
const rareCategoriesRatio = rareCategoryCount / totalCategoryCount;
|
||||||
const categoriesDocumentRatio = totalCategoryCount / categorizedDocCount;
|
const categoriesDocumentRatio = totalCategoryCount / categorizedDocCount;
|
||||||
const deadCategoriesRatio = deadCategoryCount / totalCategoryCount;
|
const deadCategoriesRatio = deadCategoryCount / totalCategoryCount;
|
||||||
|
|
|
@ -9,7 +9,9 @@ import { useLogEntryCategoriesModuleContext } from './use_log_entry_categories_m
|
||||||
|
|
||||||
export const useLogEntryCategoriesSetup = () => {
|
export const useLogEntryCategoriesSetup = () => {
|
||||||
const {
|
const {
|
||||||
|
categoryQualityWarnings,
|
||||||
cleanUpAndSetUpModule,
|
cleanUpAndSetUpModule,
|
||||||
|
fetchJobStatus,
|
||||||
lastSetupErrorMessages,
|
lastSetupErrorMessages,
|
||||||
moduleDescriptor,
|
moduleDescriptor,
|
||||||
setUpModule,
|
setUpModule,
|
||||||
|
@ -37,8 +39,10 @@ export const useLogEntryCategoriesSetup = () => {
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
|
categoryQualityWarnings,
|
||||||
cleanUpAndSetUp,
|
cleanUpAndSetUp,
|
||||||
endTime,
|
endTime,
|
||||||
|
fetchJobStatus,
|
||||||
isValidating,
|
isValidating,
|
||||||
lastSetupErrorMessages,
|
lastSetupErrorMessages,
|
||||||
moduleDescriptor,
|
moduleDescriptor,
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { i18n } from '@kbn/i18n';
|
import { i18n } from '@kbn/i18n';
|
||||||
import React, { useCallback, useEffect, useState } from 'react';
|
import React, { useCallback, useEffect } from 'react';
|
||||||
import { isJobStatusWithResults } from '../../../../common/log_analysis';
|
import { isJobStatusWithResults } from '../../../../common/log_analysis';
|
||||||
import { LoadingPage } from '../../../components/loading_page';
|
import { LoadingPage } from '../../../components/loading_page';
|
||||||
import {
|
import {
|
||||||
|
@ -14,6 +14,10 @@ import {
|
||||||
MissingSetupPrivilegesPrompt,
|
MissingSetupPrivilegesPrompt,
|
||||||
SubscriptionSplashContent,
|
SubscriptionSplashContent,
|
||||||
} from '../../../components/logging/log_analysis_setup';
|
} from '../../../components/logging/log_analysis_setup';
|
||||||
|
import {
|
||||||
|
LogAnalysisSetupFlyout,
|
||||||
|
useLogAnalysisSetupFlyoutStateContext,
|
||||||
|
} from '../../../components/logging/log_analysis_setup/setup_flyout';
|
||||||
import { SourceErrorPage } from '../../../components/source_error_page';
|
import { SourceErrorPage } from '../../../components/source_error_page';
|
||||||
import { SourceLoadingPage } from '../../../components/source_loading_page';
|
import { SourceLoadingPage } from '../../../components/source_loading_page';
|
||||||
import { useLogAnalysisCapabilitiesContext } from '../../../containers/logs/log_analysis';
|
import { useLogAnalysisCapabilitiesContext } from '../../../containers/logs/log_analysis';
|
||||||
|
@ -21,7 +25,6 @@ import { useLogEntryCategoriesModuleContext } from '../../../containers/logs/log
|
||||||
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
||||||
import { LogEntryCategoriesResultsContent } from './page_results_content';
|
import { LogEntryCategoriesResultsContent } from './page_results_content';
|
||||||
import { LogEntryCategoriesSetupContent } from './page_setup_content';
|
import { LogEntryCategoriesSetupContent } from './page_setup_content';
|
||||||
import { LogEntryCategoriesSetupFlyout } from './setup_flyout';
|
|
||||||
|
|
||||||
export const LogEntryCategoriesPageContent = () => {
|
export const LogEntryCategoriesPageContent = () => {
|
||||||
const {
|
const {
|
||||||
|
@ -40,9 +43,10 @@ export const LogEntryCategoriesPageContent = () => {
|
||||||
|
|
||||||
const { fetchJobStatus, setupStatus, jobStatus } = useLogEntryCategoriesModuleContext();
|
const { fetchJobStatus, setupStatus, jobStatus } = useLogEntryCategoriesModuleContext();
|
||||||
|
|
||||||
const [isFlyoutOpen, setIsFlyoutOpen] = useState<boolean>(false);
|
const { showModuleSetup } = useLogAnalysisSetupFlyoutStateContext();
|
||||||
const openFlyout = useCallback(() => setIsFlyoutOpen(true), []);
|
const showCategoriesModuleSetup = useCallback(() => showModuleSetup('logs_ui_categories'), [
|
||||||
const closeFlyout = useCallback(() => setIsFlyoutOpen(false), []);
|
showModuleSetup,
|
||||||
|
]);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (hasLogAnalysisReadCapabilities) {
|
if (hasLogAnalysisReadCapabilities) {
|
||||||
|
@ -71,8 +75,8 @@ export const LogEntryCategoriesPageContent = () => {
|
||||||
} else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) {
|
} else if (isJobStatusWithResults(jobStatus['log-entry-categories-count'])) {
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<LogEntryCategoriesResultsContent onOpenSetup={openFlyout} />
|
<LogEntryCategoriesResultsContent onOpenSetup={showCategoriesModuleSetup} />
|
||||||
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
|
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
} else if (!hasLogAnalysisSetupCapabilities) {
|
} else if (!hasLogAnalysisSetupCapabilities) {
|
||||||
|
@ -80,9 +84,11 @@ export const LogEntryCategoriesPageContent = () => {
|
||||||
} else {
|
} else {
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<LogEntryCategoriesSetupContent onOpenSetup={openFlyout} />
|
<LogEntryCategoriesSetupContent onOpenSetup={showCategoriesModuleSetup} />
|
||||||
<LogEntryCategoriesSetupFlyout isOpen={isFlyoutOpen} onClose={closeFlyout} />
|
<LogAnalysisSetupFlyout allowedModules={allowedSetupModules} />
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const allowedSetupModules = ['logs_ui_categories' as const];
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import React from 'react';
|
import React from 'react';
|
||||||
|
import { LogAnalysisSetupFlyoutStateProvider } from '../../../components/logging/log_analysis_setup/setup_flyout';
|
||||||
import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
|
import { LogEntryCategoriesModuleProvider } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
|
||||||
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
import { useLogSourceContext } from '../../../containers/logs/log_source';
|
||||||
import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
|
import { useActiveKibanaSpace } from '../../../hooks/use_kibana_space';
|
||||||
|
@ -27,7 +28,7 @@ export const LogEntryCategoriesPageProviders: React.FunctionComponent = ({ child
|
||||||
spaceId={space.id}
|
spaceId={space.id}
|
||||||
timestampField={sourceConfiguration.configuration.fields.timestamp}
|
timestampField={sourceConfiguration.configuration.fields.timestamp}
|
||||||
>
|
>
|
||||||
{children}
|
<LogAnalysisSetupFlyoutStateProvider>{children}</LogAnalysisSetupFlyoutStateProvider>
|
||||||
</LogEntryCategoriesModuleProvider>
|
</LogEntryCategoriesModuleProvider>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,128 +0,0 @@
|
||||||
/*
|
|
||||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
|
||||||
* or more contributor license agreements. Licensed under the Elastic License;
|
|
||||||
* you may not use this file except in compliance with the Elastic License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import {
|
|
||||||
EuiFlyout,
|
|
||||||
EuiFlyoutBody,
|
|
||||||
EuiFlyoutHeader,
|
|
||||||
EuiSpacer,
|
|
||||||
EuiSteps,
|
|
||||||
EuiText,
|
|
||||||
EuiTitle,
|
|
||||||
} from '@elastic/eui';
|
|
||||||
import { FormattedMessage } from '@kbn/i18n/react';
|
|
||||||
import React, { useCallback, useMemo } from 'react';
|
|
||||||
import {
|
|
||||||
createInitialConfigurationStep,
|
|
||||||
createProcessStep,
|
|
||||||
} from '../../../components/logging/log_analysis_setup';
|
|
||||||
import { useLogEntryCategoriesSetup } from '../../../containers/logs/log_analysis/modules/log_entry_categories';
|
|
||||||
|
|
||||||
interface LogEntryCategoriesSetupFlyoutProps {
|
|
||||||
isOpen: boolean;
|
|
||||||
onClose: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const LogEntryCategoriesSetupFlyout: React.FC<LogEntryCategoriesSetupFlyoutProps> = ({
|
|
||||||
isOpen,
|
|
||||||
onClose,
|
|
||||||
}) => {
|
|
||||||
const {
|
|
||||||
cleanUpAndSetUp,
|
|
||||||
endTime,
|
|
||||||
isValidating,
|
|
||||||
lastSetupErrorMessages,
|
|
||||||
setEndTime,
|
|
||||||
setStartTime,
|
|
||||||
setValidatedIndices,
|
|
||||||
setUp,
|
|
||||||
setupStatus,
|
|
||||||
startTime,
|
|
||||||
validatedIndices,
|
|
||||||
validationErrors,
|
|
||||||
viewResults,
|
|
||||||
} = useLogEntryCategoriesSetup();
|
|
||||||
|
|
||||||
const viewResultsAndClose = useCallback(() => {
|
|
||||||
viewResults();
|
|
||||||
onClose();
|
|
||||||
}, [viewResults, onClose]);
|
|
||||||
|
|
||||||
const steps = useMemo(
|
|
||||||
() => [
|
|
||||||
createInitialConfigurationStep({
|
|
||||||
setStartTime,
|
|
||||||
setEndTime,
|
|
||||||
startTime,
|
|
||||||
endTime,
|
|
||||||
isValidating,
|
|
||||||
validatedIndices,
|
|
||||||
setupStatus,
|
|
||||||
setValidatedIndices,
|
|
||||||
validationErrors,
|
|
||||||
}),
|
|
||||||
createProcessStep({
|
|
||||||
cleanUpAndSetUp,
|
|
||||||
errorMessages: lastSetupErrorMessages,
|
|
||||||
isConfigurationValid: validationErrors.length <= 0 && !isValidating,
|
|
||||||
setUp,
|
|
||||||
setupStatus,
|
|
||||||
viewResults: viewResultsAndClose,
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
[
|
|
||||||
cleanUpAndSetUp,
|
|
||||||
endTime,
|
|
||||||
isValidating,
|
|
||||||
lastSetupErrorMessages,
|
|
||||||
setEndTime,
|
|
||||||
setStartTime,
|
|
||||||
setUp,
|
|
||||||
setValidatedIndices,
|
|
||||||
setupStatus,
|
|
||||||
startTime,
|
|
||||||
validatedIndices,
|
|
||||||
validationErrors,
|
|
||||||
viewResultsAndClose,
|
|
||||||
]
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!isOpen) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return (
|
|
||||||
<EuiFlyout onClose={onClose}>
|
|
||||||
<EuiFlyoutHeader hasBorder>
|
|
||||||
<EuiTitle size="s">
|
|
||||||
<h3>
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.logs.setupFlyout.setupFlyoutTitle"
|
|
||||||
defaultMessage="Anomaly detection with Machine Learning"
|
|
||||||
/>
|
|
||||||
</h3>
|
|
||||||
</EuiTitle>
|
|
||||||
</EuiFlyoutHeader>
|
|
||||||
<EuiFlyoutBody>
|
|
||||||
<EuiTitle size="s">
|
|
||||||
<h3>
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.logs.setupFlyout.logCategoriesTitle"
|
|
||||||
defaultMessage="Log categories"
|
|
||||||
/>
|
|
||||||
</h3>
|
|
||||||
</EuiTitle>
|
|
||||||
<EuiText size="s">
|
|
||||||
<FormattedMessage
|
|
||||||
id="xpack.infra.logs.setupFlyout.logCategoriesDescription"
|
|
||||||
defaultMessage="Use Machine Learning to automatically categorize log messages."
|
|
||||||
/>
|
|
||||||
</EuiText>
|
|
||||||
<EuiSpacer />
|
|
||||||
<EuiSteps steps={steps} />
|
|
||||||
</EuiFlyoutBody>
|
|
||||||
</EuiFlyout>
|
|
||||||
);
|
|
||||||
};
|
|
|
@ -13,6 +13,7 @@ import { InfraBackendLibs } from './lib/infra_types';
|
||||||
import {
|
import {
|
||||||
initGetLogEntryCategoriesRoute,
|
initGetLogEntryCategoriesRoute,
|
||||||
initGetLogEntryCategoryDatasetsRoute,
|
initGetLogEntryCategoryDatasetsRoute,
|
||||||
|
initGetLogEntryCategoryDatasetsStatsRoute,
|
||||||
initGetLogEntryCategoryExamplesRoute,
|
initGetLogEntryCategoryExamplesRoute,
|
||||||
initGetLogEntryRateRoute,
|
initGetLogEntryRateRoute,
|
||||||
initGetLogEntryExamplesRoute,
|
initGetLogEntryExamplesRoute,
|
||||||
|
@ -54,6 +55,7 @@ export const initInfraServer = (libs: InfraBackendLibs) => {
|
||||||
initIpToHostName(libs);
|
initIpToHostName(libs);
|
||||||
initGetLogEntryCategoriesRoute(libs);
|
initGetLogEntryCategoriesRoute(libs);
|
||||||
initGetLogEntryCategoryDatasetsRoute(libs);
|
initGetLogEntryCategoryDatasetsRoute(libs);
|
||||||
|
initGetLogEntryCategoryDatasetsStatsRoute(libs);
|
||||||
initGetLogEntryCategoryExamplesRoute(libs);
|
initGetLogEntryCategoryExamplesRoute(libs);
|
||||||
initGetLogEntryRateRoute(libs);
|
initGetLogEntryRateRoute(libs);
|
||||||
initGetLogEntryAnomaliesRoute(libs);
|
initGetLogEntryAnomaliesRoute(libs);
|
||||||
|
|
|
@ -36,7 +36,7 @@ export async function fetchMlJob(mlAnomalyDetectors: MlAnomalyDetectors, jobId:
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
|
export const COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
|
||||||
|
|
||||||
// Finds datasets related to ML job ids
|
// Finds datasets related to ML job ids
|
||||||
export async function getLogEntryDatasets(
|
export async function getLogEntryDatasets(
|
||||||
|
|
|
@ -6,5 +6,6 @@
|
||||||
|
|
||||||
export * from './errors';
|
export * from './errors';
|
||||||
export * from './log_entry_categories_analysis';
|
export * from './log_entry_categories_analysis';
|
||||||
|
export * from './log_entry_categories_datasets_stats';
|
||||||
export * from './log_entry_rate_analysis';
|
export * from './log_entry_rate_analysis';
|
||||||
export * from './log_entry_anomalies';
|
export * from './log_entry_anomalies';
|
||||||
|
|
|
@ -0,0 +1,94 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { startTracingSpan } from '../../../common/performance_tracing';
|
||||||
|
import { decodeOrThrow } from '../../../common/runtime_types';
|
||||||
|
import type { MlAnomalyDetectors, MlSystem } from '../../types';
|
||||||
|
import { COMPOSITE_AGGREGATION_BATCH_SIZE } from './common';
|
||||||
|
import {
|
||||||
|
CompositeDatasetKey,
|
||||||
|
createLatestLogEntryCategoriesDatasetsStatsQuery,
|
||||||
|
latestLogEntryCategoriesDatasetsStatsResponseRT,
|
||||||
|
LogEntryCategoryDatasetStatsBucket,
|
||||||
|
} from './queries/latest_log_entry_categories_datasets_stats';
|
||||||
|
|
||||||
|
export async function getLatestLogEntriesCategoriesDatasetsStats(
|
||||||
|
context: {
|
||||||
|
infra: {
|
||||||
|
mlAnomalyDetectors: MlAnomalyDetectors;
|
||||||
|
mlSystem: MlSystem;
|
||||||
|
};
|
||||||
|
},
|
||||||
|
jobIds: string[],
|
||||||
|
startTime: number,
|
||||||
|
endTime: number,
|
||||||
|
includeCategorizerStatuses: Array<'ok' | 'warn'> = []
|
||||||
|
) {
|
||||||
|
const finalizeLogEntryCategoriesDatasetsStats = startTracingSpan('get categories datasets stats');
|
||||||
|
|
||||||
|
let latestLogEntryCategoriesDatasetsStatsBuckets: LogEntryCategoryDatasetStatsBucket[] = [];
|
||||||
|
let afterLatestBatchKey: CompositeDatasetKey | undefined;
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const latestLogEntryCategoriesDatasetsStatsResponse = await context.infra.mlSystem.mlAnomalySearch(
|
||||||
|
createLatestLogEntryCategoriesDatasetsStatsQuery(
|
||||||
|
jobIds,
|
||||||
|
startTime,
|
||||||
|
endTime,
|
||||||
|
COMPOSITE_AGGREGATION_BATCH_SIZE,
|
||||||
|
afterLatestBatchKey
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const { after_key: afterKey, buckets: latestBatchBuckets = [] } =
|
||||||
|
decodeOrThrow(latestLogEntryCategoriesDatasetsStatsResponseRT)(
|
||||||
|
latestLogEntryCategoriesDatasetsStatsResponse
|
||||||
|
).aggregations?.dataset_composite_terms ?? {};
|
||||||
|
|
||||||
|
const latestIncludedBatchBuckets =
|
||||||
|
includeCategorizerStatuses.length > 0
|
||||||
|
? latestBatchBuckets.filter((bucket) =>
|
||||||
|
bucket.categorizer_stats_top_hits.hits.hits.some((hit) =>
|
||||||
|
includeCategorizerStatuses.includes(hit._source.categorization_status)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
: latestBatchBuckets;
|
||||||
|
|
||||||
|
latestLogEntryCategoriesDatasetsStatsBuckets = [
|
||||||
|
...latestLogEntryCategoriesDatasetsStatsBuckets,
|
||||||
|
...latestIncludedBatchBuckets,
|
||||||
|
];
|
||||||
|
|
||||||
|
afterLatestBatchKey = afterKey;
|
||||||
|
if (afterKey == null || latestBatchBuckets.length < COMPOSITE_AGGREGATION_BATCH_SIZE) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const logEntryCategoriesDatasetsStatsSpan = finalizeLogEntryCategoriesDatasetsStats();
|
||||||
|
|
||||||
|
return {
|
||||||
|
data: latestLogEntryCategoriesDatasetsStatsBuckets.map((bucket) => {
|
||||||
|
const latestHitSource = bucket.categorizer_stats_top_hits.hits.hits[0]._source;
|
||||||
|
|
||||||
|
return {
|
||||||
|
categorization_status: latestHitSource.categorization_status,
|
||||||
|
categorized_doc_count: latestHitSource.categorized_doc_count,
|
||||||
|
dataset: bucket.key.dataset ?? '',
|
||||||
|
dead_category_count: latestHitSource.dead_category_count,
|
||||||
|
failed_category_count: latestHitSource.failed_category_count,
|
||||||
|
frequent_category_count: latestHitSource.frequent_category_count,
|
||||||
|
job_id: latestHitSource.job_id,
|
||||||
|
log_time: latestHitSource.log_time,
|
||||||
|
rare_category_count: latestHitSource.rare_category_count,
|
||||||
|
total_category_count: latestHitSource.total_category_count,
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
timing: {
|
||||||
|
spans: [logEntryCategoriesDatasetsStatsSpan],
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
|
@ -40,7 +40,20 @@ export const createTimeRangeFilters = (startTime: number, endTime: number) => [
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
export const createResultTypeFilters = (resultTypes: Array<'model_plot' | 'record'>) => [
|
export const createLogTimeRangeFilters = (startTime: number, endTime: number) => [
|
||||||
|
{
|
||||||
|
range: {
|
||||||
|
log_time: {
|
||||||
|
gte: startTime,
|
||||||
|
lte: endTime,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export const createResultTypeFilters = (
|
||||||
|
resultTypes: Array<'categorizer_stats' | 'model_plot' | 'record'>
|
||||||
|
) => [
|
||||||
{
|
{
|
||||||
terms: {
|
terms: {
|
||||||
result_type: resultTypes,
|
result_type: resultTypes,
|
||||||
|
|
|
@ -0,0 +1,133 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as rt from 'io-ts';
|
||||||
|
import { commonSearchSuccessResponseFieldsRT } from '../../../utils/elasticsearch_runtime_types';
|
||||||
|
import {
|
||||||
|
createJobIdsFilters,
|
||||||
|
createResultTypeFilters,
|
||||||
|
defaultRequestParameters,
|
||||||
|
createLogTimeRangeFilters,
|
||||||
|
} from './common';
|
||||||
|
|
||||||
|
export const createLatestLogEntryCategoriesDatasetsStatsQuery = (
|
||||||
|
logEntryCategoriesJobIds: string[],
|
||||||
|
startTime: number,
|
||||||
|
endTime: number,
|
||||||
|
size: number,
|
||||||
|
afterKey?: CompositeDatasetKey
|
||||||
|
) => ({
|
||||||
|
...defaultRequestParameters,
|
||||||
|
body: {
|
||||||
|
query: {
|
||||||
|
bool: {
|
||||||
|
filter: [
|
||||||
|
...createJobIdsFilters(logEntryCategoriesJobIds),
|
||||||
|
...createResultTypeFilters(['categorizer_stats']),
|
||||||
|
...createLogTimeRangeFilters(startTime, endTime),
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
aggregations: {
|
||||||
|
dataset_composite_terms: {
|
||||||
|
composite: {
|
||||||
|
after: afterKey,
|
||||||
|
size,
|
||||||
|
sources: [
|
||||||
|
{
|
||||||
|
dataset: {
|
||||||
|
terms: {
|
||||||
|
field: 'partition_field_value',
|
||||||
|
missing_bucket: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
aggs: {
|
||||||
|
categorizer_stats_top_hits: {
|
||||||
|
top_hits: {
|
||||||
|
size: 1,
|
||||||
|
sort: [
|
||||||
|
{
|
||||||
|
log_time: 'desc',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
_source: [
|
||||||
|
'categorization_status',
|
||||||
|
'categorized_doc_count',
|
||||||
|
'dead_category_count',
|
||||||
|
'failed_category_count',
|
||||||
|
'frequent_category_count',
|
||||||
|
'job_id',
|
||||||
|
'log_time',
|
||||||
|
'rare_category_count',
|
||||||
|
'total_category_count',
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
size: 0,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const logEntryCategoryStatusRT = rt.keyof({
|
||||||
|
ok: null,
|
||||||
|
warn: null,
|
||||||
|
});
|
||||||
|
|
||||||
|
export const logEntryCategorizerStatsHitRT = rt.type({
|
||||||
|
_source: rt.type({
|
||||||
|
categorization_status: logEntryCategoryStatusRT,
|
||||||
|
categorized_doc_count: rt.number,
|
||||||
|
dead_category_count: rt.number,
|
||||||
|
failed_category_count: rt.number,
|
||||||
|
frequent_category_count: rt.number,
|
||||||
|
job_id: rt.string,
|
||||||
|
log_time: rt.number,
|
||||||
|
rare_category_count: rt.number,
|
||||||
|
total_category_count: rt.number,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type LogEntryCategorizerStatsHit = rt.TypeOf<typeof logEntryCategorizerStatsHitRT>;
|
||||||
|
|
||||||
|
const compositeDatasetKeyRT = rt.type({
|
||||||
|
dataset: rt.union([rt.string, rt.null]),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type CompositeDatasetKey = rt.TypeOf<typeof compositeDatasetKeyRT>;
|
||||||
|
|
||||||
|
const logEntryCategoryDatasetStatsBucketRT = rt.type({
|
||||||
|
key: compositeDatasetKeyRT,
|
||||||
|
categorizer_stats_top_hits: rt.type({
|
||||||
|
hits: rt.type({
|
||||||
|
hits: rt.array(logEntryCategorizerStatsHitRT),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type LogEntryCategoryDatasetStatsBucket = rt.TypeOf<
|
||||||
|
typeof logEntryCategoryDatasetStatsBucketRT
|
||||||
|
>;
|
||||||
|
|
||||||
|
export const latestLogEntryCategoriesDatasetsStatsResponseRT = rt.intersection([
|
||||||
|
commonSearchSuccessResponseFieldsRT,
|
||||||
|
rt.partial({
|
||||||
|
aggregations: rt.type({
|
||||||
|
dataset_composite_terms: rt.type({
|
||||||
|
after_key: compositeDatasetKeyRT,
|
||||||
|
buckets: rt.array(logEntryCategoryDatasetStatsBucketRT),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
]);
|
||||||
|
|
||||||
|
export type LatestLogEntryCategoriesDatasetsStatsResponse = rt.TypeOf<
|
||||||
|
typeof latestLogEntryCategoriesDatasetsStatsResponseRT
|
||||||
|
>;
|
|
@ -6,6 +6,7 @@
|
||||||
|
|
||||||
export * from './log_entry_categories';
|
export * from './log_entry_categories';
|
||||||
export * from './log_entry_category_datasets';
|
export * from './log_entry_category_datasets';
|
||||||
|
export * from './log_entry_category_datasets_stats';
|
||||||
export * from './log_entry_category_examples';
|
export * from './log_entry_category_examples';
|
||||||
export * from './log_entry_rate';
|
export * from './log_entry_rate';
|
||||||
export * from './log_entry_examples';
|
export * from './log_entry_examples';
|
||||||
|
|
|
@ -0,0 +1,79 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import Boom from 'boom';
|
||||||
|
import {
|
||||||
|
getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT,
|
||||||
|
getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT,
|
||||||
|
LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||||
|
} from '../../../../common/http_api/log_analysis';
|
||||||
|
import { createValidationFunction } from '../../../../common/runtime_types';
|
||||||
|
import type { InfraBackendLibs } from '../../../lib/infra_types';
|
||||||
|
import { getLatestLogEntriesCategoriesDatasetsStats } from '../../../lib/log_analysis';
|
||||||
|
import { isMlPrivilegesError } from '../../../lib/log_analysis/errors';
|
||||||
|
import { assertHasInfraMlPlugins } from '../../../utils/request_context';
|
||||||
|
|
||||||
|
export const initGetLogEntryCategoryDatasetsStatsRoute = ({ framework }: InfraBackendLibs) => {
|
||||||
|
framework.registerRoute(
|
||||||
|
{
|
||||||
|
method: 'post',
|
||||||
|
path: LOG_ANALYSIS_GET_LATEST_LOG_ENTRY_CATEGORY_DATASETS_STATS_PATH,
|
||||||
|
validate: {
|
||||||
|
body: createValidationFunction(getLatestLogEntryCategoryDatasetsStatsRequestPayloadRT),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
framework.router.handleLegacyErrors(async (requestContext, request, response) => {
|
||||||
|
const {
|
||||||
|
data: {
|
||||||
|
jobIds,
|
||||||
|
timeRange: { startTime, endTime },
|
||||||
|
includeCategorizerStatuses,
|
||||||
|
},
|
||||||
|
} = request.body;
|
||||||
|
|
||||||
|
try {
|
||||||
|
assertHasInfraMlPlugins(requestContext);
|
||||||
|
|
||||||
|
const { data: datasetStats, timing } = await getLatestLogEntriesCategoriesDatasetsStats(
|
||||||
|
requestContext,
|
||||||
|
jobIds,
|
||||||
|
startTime,
|
||||||
|
endTime,
|
||||||
|
includeCategorizerStatuses
|
||||||
|
);
|
||||||
|
|
||||||
|
return response.ok({
|
||||||
|
body: getLatestLogEntryCategoryDatasetsStatsSuccessResponsePayloadRT.encode({
|
||||||
|
data: {
|
||||||
|
datasetStats,
|
||||||
|
},
|
||||||
|
timing,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (Boom.isBoom(error)) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isMlPrivilegesError(error)) {
|
||||||
|
return response.customError({
|
||||||
|
statusCode: 403,
|
||||||
|
body: {
|
||||||
|
message: error.message,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.customError({
|
||||||
|
statusCode: error.statusCode ?? 500,
|
||||||
|
body: {
|
||||||
|
message: error.message ?? 'An unexpected error occurred',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
};
|
|
@ -14,7 +14,11 @@
|
||||||
"use_null": true
|
"use_null": true
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"influencers": ["event.dataset", "mlcategory"]
|
"influencers": ["event.dataset", "mlcategory"],
|
||||||
|
"per_partition_categorization": {
|
||||||
|
"enabled": true,
|
||||||
|
"stop_on_warn": false
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"analysis_limits": {
|
"analysis_limits": {
|
||||||
"model_memory_limit": "100mb",
|
"model_memory_limit": "100mb",
|
||||||
|
@ -29,6 +33,6 @@
|
||||||
},
|
},
|
||||||
"custom_settings": {
|
"custom_settings": {
|
||||||
"created_by": "ml-module-logs-ui-categories",
|
"created_by": "ml-module-logs-ui-categories",
|
||||||
"job_revision": 0
|
"job_revision": 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -8486,9 +8486,6 @@
|
||||||
"xpack.infra.logs.search.searchInLogsAriaLabel": "検索",
|
"xpack.infra.logs.search.searchInLogsAriaLabel": "検索",
|
||||||
"xpack.infra.logs.search.searchInLogsPlaceholder": "検索",
|
"xpack.infra.logs.search.searchInLogsPlaceholder": "検索",
|
||||||
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 件のハイライトされたエントリー} other {# 件のハイライトされたエントリー}}",
|
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 件のハイライトされたエントリー} other {# 件のハイライトされたエントリー}}",
|
||||||
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "機械学習を使用して、ログメッセージを自動的に分類します。",
|
|
||||||
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "ログカテゴリー",
|
|
||||||
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "機械学習を使用した異常検知",
|
|
||||||
"xpack.infra.logs.showingEntriesFromTimestamp": "{timestamp} 以降のエントリーを表示中",
|
"xpack.infra.logs.showingEntriesFromTimestamp": "{timestamp} 以降のエントリーを表示中",
|
||||||
"xpack.infra.logs.showingEntriesUntilTimestamp": "{timestamp} までのエントリーを表示中",
|
"xpack.infra.logs.showingEntriesUntilTimestamp": "{timestamp} までのエントリーを表示中",
|
||||||
"xpack.infra.logs.startStreamingButtonLabel": "ライブストリーム",
|
"xpack.infra.logs.startStreamingButtonLabel": "ライブストリーム",
|
||||||
|
|
|
@ -8491,9 +8491,6 @@
|
||||||
"xpack.infra.logs.search.searchInLogsAriaLabel": "搜索",
|
"xpack.infra.logs.search.searchInLogsAriaLabel": "搜索",
|
||||||
"xpack.infra.logs.search.searchInLogsPlaceholder": "搜索",
|
"xpack.infra.logs.search.searchInLogsPlaceholder": "搜索",
|
||||||
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 个高亮条目} other {# 个高亮条目}}",
|
"xpack.infra.logs.searchResultTooltip": "{bucketCount, plural, one {# 个高亮条目} other {# 个高亮条目}}",
|
||||||
"xpack.infra.logs.setupFlyout.logCategoriesDescription": "使用 Machine Learning 自动归类日志消息。",
|
|
||||||
"xpack.infra.logs.setupFlyout.logCategoriesTitle": "日志类别",
|
|
||||||
"xpack.infra.logs.setupFlyout.setupFlyoutTitle": "通过 Machine Learning 检测异常",
|
|
||||||
"xpack.infra.logs.showingEntriesFromTimestamp": "正在显示自 {timestamp} 起的条目",
|
"xpack.infra.logs.showingEntriesFromTimestamp": "正在显示自 {timestamp} 起的条目",
|
||||||
"xpack.infra.logs.showingEntriesUntilTimestamp": "正在显示截止于 {timestamp} 的条目",
|
"xpack.infra.logs.showingEntriesUntilTimestamp": "正在显示截止于 {timestamp} 的条目",
|
||||||
"xpack.infra.logs.startStreamingButtonLabel": "实时流式传输",
|
"xpack.infra.logs.startStreamingButtonLabel": "实时流式传输",
|
||||||
|
|
Loading…
Reference in a new issue