maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Improve messaging and support for datafeed using aggregated and scripted fields (#84594)

Browse source
This commit is contained in:
Quynh Nguyen 2020-12-10 11:35:51 -06:00 committed by GitHub
parent 1b5d43b2e2
commit 008a420f81
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 822 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts
View file

@ -28,6 +28,7 @@ export interface MlSummaryJob {
nodeName?: string;
auditMessage?: Partial<AuditMessage>;
isSingleMetricViewerJob: boolean;
isNotSingleMetricViewerJobMessage?: string;
deleting?: boolean;
latestTimestampSortValue?: number;
earliestStartTimestampMs?: number;
@ -45,6 +46,8 @@ export interface AuditMessage {
export type MlSummaryJobs = MlSummaryJob[];
export interface MlJobWithTimeRange extends CombinedJobWithStats {
id: string;
isNotSingleMetricViewerJobMessage?: string;
timeRange: {
from: number;
to: number;

10
x-pack/plugins/ml/common/util/datafeed_utils.ts
View file

@ -6,12 +6,16 @@
import { Aggregation, Datafeed } from '../types/anomaly_detection_jobs';
export function getAggregations<T>(obj: any): T | undefined {
if (obj?.aggregations !== undefined) return obj.aggregations;
if (obj?.aggs !== undefined) return obj.aggs;
return undefined;
}
export const getDatafeedAggregations = (
datafeedConfig: Partial<Datafeed> | undefined
): Aggregation | undefined => {
if (datafeedConfig?.aggregations !== undefined) return datafeedConfig.aggregations;
if (datafeedConfig?.aggs !== undefined) return datafeedConfig.aggs;
return undefined;
return getAggregations<Aggregation>(datafeedConfig);
};
export const getAggregationBucketsName = (aggregations: any): string | undefined => {

58
x-pack/plugins/ml/common/util/job_utils.ts
View file

@ -10,6 +10,7 @@ import moment, { Duration } from 'moment';
// @ts-ignore
import numeral from '@elastic/numeral';
import { i18n } from '@kbn/i18n';
import { ALLOWED_DATA_UNITS, JOB_ID_MAX_LENGTH } from '../constants/validation';
import { parseInterval } from './parse_interval';
import { maxLengthValidator } from './validators';
@ -20,7 +21,12 @@ import { MlServerLimits } from '../types/ml_server_info';
import { JobValidationMessage, JobValidationMessageId } from '../constants/messages';
import { ES_AGGREGATION, ML_JOB_AGGREGATION } from '../constants/aggregation_types';
import { MLCATEGORY } from '../constants/field_types';
import { getDatafeedAggregations } from './datafeed_utils';
import {
getAggregationBucketsName,
getAggregations,
getDatafeedAggregations,
} from './datafeed_utils';
import { findAggField } from './validation_utils';
export interface ValidationResults {
valid: boolean;
@ -43,20 +49,8 @@ export function calculateDatafeedFrequencyDefaultSeconds(bucketSpanSeconds: numb
return freq;
}
// Returns a flag to indicate whether the job is suitable for viewing
// in the Time Series dashboard.
export function isTimeSeriesViewJob(job: CombinedJob): boolean {
// only allow jobs with at least one detector whose function corresponds to
// an ES aggregation which can be viewed in the single metric view and which
// doesn't use a scripted field which can be very difficult or impossible to
// invert to a reverse search, or when model plot has been enabled.
for (let i = 0; i < job.analysis_config.detectors.length; i++) {
if (isTimeSeriesViewDetector(job, i)) {
return true;
}
}
return false;
return getSingleMetricViewerJobErrorMessage(job) === undefined;
}
// Returns a flag to indicate whether the detector at the index in the specified job
@ -99,6 +93,24 @@ export function isSourceDataChartableForDetector(job: CombinedJob, detectorIndex
scriptFields.indexOf(dtr.by_field_name!) === -1 &&
scriptFields.indexOf(dtr.over_field_name!) === -1;
}
// We cannot plot the source data for some specific aggregation configurations
const hasDatafeed =
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
if (hasDatafeed) {
const aggs = getDatafeedAggregations(job.datafeed_config);
if (aggs !== undefined) {
const aggBucketsName = getAggregationBucketsName(aggs);
if (aggBucketsName !== undefined) {
// if fieldName is a aggregated field under nested terms using bucket_script
const aggregations = getAggregations<{ [key: string]: any }>(aggs[aggBucketsName]) ?? {};
const foundField = findAggField(aggregations, dtr.field_name, false);
if (foundField?.bucket_script !== undefined) {
return false;
}
}
}
}
}
return isSourceDataChartable;
@ -134,6 +146,24 @@ export function isModelPlotChartableForDetector(job: Job, detectorIndex: number)
return isModelPlotChartable;
}
// Returns a reason to indicate why the job configuration is not supported
// if the result is undefined, that means the single metric job should be viewable
export function getSingleMetricViewerJobErrorMessage(job: CombinedJob): string | undefined {
// only allow jobs with at least one detector whose function corresponds to
// an ES aggregation which can be viewed in the single metric view and which
// doesn't use a scripted field which can be very difficult or impossible to
// invert to a reverse search, or when model plot has been enabled.
const isChartableTimeSeriesViewJob = job.analysis_config.detectors.some((detector, idx) =>
isTimeSeriesViewDetector(job, idx)
);
if (isChartableTimeSeriesViewJob === false) {
return i18n.translate('xpack.ml.timeSeriesJob.notViewableTimeSeriesJobMessage', {
defaultMessage: 'not a viewable time series job',
});
}
}
// Returns the names of the partition, by, and over fields for the detector with the
// specified index from the supplied ML job configuration.
export function getPartitioningFieldNames(job: CombinedJob, detectorIndex: number): string[] {

11
x-pack/plugins/ml/common/util/validation_utils.ts
View file

@ -32,15 +32,20 @@ export function isValidJson(json: string) {
}
}
export function findAggField(aggs: Record<string, any>, fieldName: string): any {
export function findAggField(
aggs: Record<string, any> | { [key: string]: any },
fieldName: string | undefined,
returnParent: boolean = false
): any {
if (fieldName === undefined) return;
let value;
Object.keys(aggs).some(function (k) {
if (k === fieldName) {
value = aggs[k];
value = returnParent === true ? aggs : aggs[k];
return true;
}
if (aggs.hasOwnProperty(k) && typeof aggs[k] === 'object') {
value = findAggField(aggs[k], fieldName);
value = findAggField(aggs[k], fieldName, returnParent);
return value !== undefined;
}
});

3
x-pack/plugins/ml/public/application/components/custom_hooks/use_create_ad_links.ts
View file

@ -22,13 +22,14 @@ export const useCreateADLinks = () => {
const userTimeSettings = useUiSettings().get(ANOMALY_DETECTION_DEFAULT_TIME_RANGE);
const createLinkWithUserDefaults = useCallback(
(location, jobList) => {
return mlJobService.createResultsUrlForJobs(
const resultsUrl = mlJobService.createResultsUrlForJobs(
jobList,
location,
useUserTimeSettings === true && userTimeSettings !== undefined
? userTimeSettings
: undefined
);
return `${basePath.get()}/app/ml/${resultsUrl}`;
},
[basePath]
);

18
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/back_to_list_panel/back_to_list_panel.tsx
View file

@ -7,21 +7,13 @@
import React, { FC, Fragment } from 'react';
import { EuiCard, EuiIcon } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useMlKibana, useMlUrlGenerator } from '../../../../../contexts/kibana';
import { useMlLink } from '../../../../../contexts/kibana';
import { ML_PAGES } from '../../../../../../../common/constants/ml_url_generator';
export const BackToListPanel: FC = () => {
const urlGenerator = useMlUrlGenerator();
const {
services: {
application: { navigateToUrl },
},
} = useMlKibana();
const redirectToAnalyticsManagementPage = async () => {
const url = await urlGenerator.createUrl({ page: ML_PAGES.DATA_FRAME_ANALYTICS_JOBS_MANAGE });
await navigateToUrl(url);
};
const analyticsManagementPageLink = useMlLink({
page: ML_PAGES.DATA_FRAME_ANALYTICS_JOBS_MANAGE,
});
return (
<Fragment>
@ -37,7 +29,7 @@ export const BackToListPanel: FC = () => {
defaultMessage: 'Return to the analytics management page.',
}
)}
onClick={redirectToAnalyticsManagementPage}
href={analyticsManagementPageLink}
data-test-subj="analyticsWizardCardManagement"
/>
</Fragment>

25
x-pack/plugins/ml/public/application/data_frame_analytics/pages/analytics_creation/components/view_results_panel/view_results_panel.tsx
View file

@ -7,9 +7,8 @@
import React, { FC, Fragment } from 'react';
import { EuiCard, EuiIcon } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useMlUrlGenerator } from '../../../../../contexts/kibana';
import { useMlLink } from '../../../../../contexts/kibana';
import { ML_PAGES } from '../../../../../../../common/constants/ml_url_generator';
import { useNavigateToPath } from '../../../../../contexts/kibana';
import { DataFrameAnalysisConfigType } from '../../../../../../../common/types/data_frame_analytics';
interface Props {
jobId: string;
@ -17,19 +16,13 @@ interface Props {
}
export const ViewResultsPanel: FC<Props> = ({ jobId, analysisType }) => {
const urlGenerator = useMlUrlGenerator();
const navigateToPath = useNavigateToPath();
const redirectToAnalyticsExplorationPage = async () => {
const path = await urlGenerator.createUrl({
page: ML_PAGES.DATA_FRAME_ANALYTICS_EXPLORATION,
pageState: {
jobId,
analysisType,
},
});
await navigateToPath(path);
};
const analyticsExplorationPageLink = useMlLink({
page: ML_PAGES.DATA_FRAME_ANALYTICS_EXPLORATION,
pageState: {
jobId,
analysisType,
},
});
return (
<Fragment>
@ -45,7 +38,7 @@ export const ViewResultsPanel: FC<Props> = ({ jobId, analysisType }) => {
defaultMessage: 'View results for the analytics job.',
}
)}
onClick={redirectToAnalyticsExplorationPage}
href={analyticsExplorationPageLink}
data-test-subj="analyticsWizardViewResultsCard"
/>
</Fragment>

2
x-pack/plugins/ml/public/application/explorer/explorer_charts/__snapshots__/explorer_charts_container_service.test.js.snap generated
View file

@ -3,6 +3,7 @@
exports[`explorerChartsContainerService call anomalyChangeListener with actual series config 1`] = `
Object {
"chartsPerRow": 1,
"errorMessages": Object {},
"seriesToPlot": Array [
Object {
"bucketSpanSeconds": 900,
@ -63,6 +64,7 @@ Object {
exports[`explorerChartsContainerService call anomalyChangeListener with actual series config 2`] = `
Object {
"chartsPerRow": 1,
"errorMessages": Object {},
"seriesToPlot": Array [
Object {
"bucketSpanSeconds": 900,

44
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container.js
View file

@ -31,6 +31,7 @@ import { MlTooltipComponent } from '../../components/chart_tooltip';
import { withKibana } from '../../../../../../../src/plugins/kibana_react/public';
import { ML_APP_URL_GENERATOR } from '../../../../common/constants/ml_url_generator';
import { addItemToRecentlyAccessed } from '../../util/recently_accessed';
import { ExplorerChartsErrorCallOuts } from './explorer_charts_error_callouts';
const textTooManyBuckets = i18n.translate('xpack.ml.explorer.charts.tooManyBucketsDescription', {
defaultMessage:
@ -165,6 +166,7 @@ export const ExplorerChartsContainerUI = ({
severity,
tooManyBuckets,
kibana,
errorMessages,
}) => {
const {
services: {
@ -183,27 +185,29 @@ export const ExplorerChartsContainerUI = ({
const chartsColumns = chartsPerRow === 1 ? 0 : chartsPerRow;
const wrapLabel = seriesToPlot.some((series) => isLabelLengthAboveThreshold(series));
return (
<EuiFlexGrid columns={chartsColumns}>
{seriesToPlot.length > 0 &&
seriesToPlot.map((series) => (
<EuiFlexItem
key={getChartId(series)}
className="ml-explorer-chart-container"
style={{ minWidth: chartsWidth }}
>
<ExplorerChartContainer
series={series}
severity={severity}
tooManyBuckets={tooManyBuckets}
wrapLabel={wrapLabel}
navigateToApp={navigateToApp}
mlUrlGenerator={mlUrlGenerator}
/>
</EuiFlexItem>
))}
</EuiFlexGrid>
<>
<ExplorerChartsErrorCallOuts errorMessagesByType={errorMessages} />
<EuiFlexGrid columns={chartsColumns}>
{seriesToPlot.length > 0 &&
seriesToPlot.map((series) => (
<EuiFlexItem
key={getChartId(series)}
className="ml-explorer-chart-container"
style={{ minWidth: chartsWidth }}
>
<ExplorerChartContainer
series={series}
severity={severity}
tooManyBuckets={tooManyBuckets}
wrapLabel={wrapLabel}
navigateToApp={navigateToApp}
mlUrlGenerator={mlUrlGenerator}
/>
</EuiFlexItem>
))}
</EuiFlexGrid>
</>
);
};

6
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.d.ts vendored
View file

@ -4,11 +4,17 @@
* you may not use this file except in compliance with the Elastic License.
*/
import type { JobId } from '../../../../common/types/anomaly_detection_jobs';
export interface ExplorerChartSeriesErrorMessages {
[key: string]: Set<JobId>;
}
export declare interface ExplorerChartsData {
chartsPerRow: number;
seriesToPlot: any[];
tooManyBuckets: boolean;
timeFieldName: string;
errorMessages: ExplorerChartSeriesErrorMessages;
}
export declare const getDefaultChartsData: () => ExplorerChartsData;

99
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_container_service.js
View file

@ -28,10 +28,12 @@ import { mlJobService } from '../../services/job_service';
import { explorerService } from '../explorer_dashboard_service';
import { CHART_TYPE } from '../explorer_constants';
import { i18n } from '@kbn/i18n';
export function getDefaultChartsData() {
return {
chartsPerRow: 1,
errorMessages: undefined,
seriesToPlot: [],
// default values, will update on every re-render
tooManyBuckets: false,
@ -58,7 +60,7 @@ export const anomalyDataChange = function (
const filteredRecords = anomalyRecords.filter((record) => {
return Number(record.record_score) >= severity;
});
const allSeriesRecords = processRecordsForDisplay(filteredRecords);
const [allSeriesRecords, errorMessages] = processRecordsForDisplay(filteredRecords);
// Calculate the number of charts per row, depending on the width available, to a max of 4.
let chartsPerRow = Math.min(
Math.max(Math.floor(chartsContainerWidth / 550), 1),
@ -97,10 +99,12 @@ export const anomalyDataChange = function (
chartData: null,
}));
data.errorMessages = errorMessages;
explorerService.setCharts({ ...data });
if (seriesConfigs.length === 0) {
return;
return data;
}
// Query 1 - load the raw metric data.
@ -109,7 +113,9 @@ export const anomalyDataChange = function (
const job = mlJobService.getJob(jobId);
// If source data can be plotted, use that, otherwise model plot will be available.
// If the job uses aggregation or scripted fields, and if it's a config we don't support
// use model plot data if model plot is enabled
// else if source data can be plotted, use that, otherwise model plot will be available.
const useSourceData = isSourceDataChartableForDetector(job, detectorIndex);
if (useSourceData === true) {
const datafeedQuery = get(config, 'datafeedConfig.query', null);
@ -422,21 +428,50 @@ export const anomalyDataChange = function (
function processRecordsForDisplay(anomalyRecords) {
// Aggregate the anomaly data by detector, and entity (by/over/partition).
if (anomalyRecords.length === 0) {
return [];
return [[], undefined];
}
// Aggregate by job, detector, and analysis fields (partition, by, over).
const aggregatedData = {};
const jobsErrorMessage = {};
each(anomalyRecords, (record) => {
// Check if we can plot a chart for this record, depending on whether the source data
// is chartable, and if model plot is enabled for the job.
const job = mlJobService.getJob(record.job_id);
// if we already know this job has datafeed aggregations we cannot support
// no need to do more checks
if (jobsErrorMessage[record.job_id] !== undefined) {
return;
}
let isChartable = isSourceDataChartableForDetector(job, record.detector_index);
if (isChartable === false && isModelPlotChartableForDetector(job, record.detector_index)) {
// Check if model plot is enabled for this job.
// Need to check the entity fields for the record in case the model plot config has a terms list.
const entityFields = getEntityFieldList(record);
isChartable = isModelPlotEnabled(job, record.detector_index, entityFields);
if (isChartable === false) {
if (isModelPlotChartableForDetector(job, record.detector_index)) {
// Check if model plot is enabled for this job.
// Need to check the entity fields for the record in case the model plot config has a terms list.
const entityFields = getEntityFieldList(record);
if (isModelPlotEnabled(job, record.detector_index, entityFields)) {
isChartable = true;
} else {
isChartable = false;
jobsErrorMessage[record.job_id] = i18n.translate(
'xpack.ml.timeSeriesJob.sourceDataNotChartableWithDisabledModelPlotMessage',
{
defaultMessage:
'source data is not viewable for this detector and model plot is disabled',
}
);
}
} else {
jobsErrorMessage[record.job_id] = i18n.translate(
'xpack.ml.timeSeriesJob.sourceDataModelPlotNotChartableMessage',
{
defaultMessage: 'both source data and model plot are not chartable for this detector',
}
);
}
}
if (isChartable === false) {
@ -529,34 +564,48 @@ function processRecordsForDisplay(anomalyRecords) {
}
});
// Group job id by error message instead of by job:
const errorMessages = {};
Object.keys(jobsErrorMessage).forEach((jobId) => {
const msg = jobsErrorMessage[jobId];
if (errorMessages[msg] === undefined) {
errorMessages[msg] = new Set([jobId]);
} else {
errorMessages[msg].add(jobId);
}
});
let recordsForSeries = [];
// Convert to an array of the records with the highest record_score per unique series.
each(aggregatedData, (detectorsForJob) => {
each(detectorsForJob, (groupsForDetector) => {
if (groupsForDetector.maxScoreRecord !== undefined) {
// Detector with no partition / by field.
recordsForSeries.push(groupsForDetector.maxScoreRecord);
if (groupsForDetector.errorMessage !== undefined) {
recordsForSeries.push(groupsForDetector.errorMessage);
} else {
each(groupsForDetector, (valuesForGroup) => {
each(valuesForGroup, (dataForGroupValue) => {
if (dataForGroupValue.maxScoreRecord !== undefined) {
recordsForSeries.push(dataForGroupValue.maxScoreRecord);
} else {
// Second level of aggregation for partition and by/over.
each(dataForGroupValue, (splitsForGroup) => {
each(splitsForGroup, (dataForSplitValue) => {
recordsForSeries.push(dataForSplitValue.maxScoreRecord);
if (groupsForDetector.maxScoreRecord !== undefined) {
// Detector with no partition / by field.
recordsForSeries.push(groupsForDetector.maxScoreRecord);
} else {
each(groupsForDetector, (valuesForGroup) => {
each(valuesForGroup, (dataForGroupValue) => {
if (dataForGroupValue.maxScoreRecord !== undefined) {
recordsForSeries.push(dataForGroupValue.maxScoreRecord);
} else {
// Second level of aggregation for partition and by/over.
each(dataForGroupValue, (splitsForGroup) => {
each(splitsForGroup, (dataForSplitValue) => {
recordsForSeries.push(dataForSplitValue.maxScoreRecord);
});
});
});
}
}
});
});
});
}
}
});
});
recordsForSeries = sortBy(recordsForSeries, 'record_score').reverse();
return recordsForSeries;
return [recordsForSeries, errorMessages];
}
function calculateChartRange(

35
x-pack/plugins/ml/public/application/explorer/explorer_charts/explorer_charts_error_callouts.tsx Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { EuiCallOut, EuiSpacer } from '@elastic/eui';
import { FormattedMessage } from '@kbn/i18n/react';
import React, { FC } from 'react';
import { ExplorerChartSeriesErrorMessages } from './explorer_charts_container_service';
interface ExplorerChartsErrorCalloutsProps {
errorMessagesByType: ExplorerChartSeriesErrorMessages;
}
export const ExplorerChartsErrorCallOuts: FC<ExplorerChartsErrorCalloutsProps> = ({
errorMessagesByType,
}) => {
if (!errorMessagesByType || Object.keys(errorMessagesByType).length === 0) return null;
const content = Object.keys(errorMessagesByType).map((errorType) => (
<EuiCallOut color={'warning'} size="s" key={errorType}>
<FormattedMessage
id="xpack.ml.explorerCharts.errorCallOutMessage"
defaultMessage="You can't view anomaly charts for {jobs} because {reason}."
values={{ jobs: [...errorMessagesByType[errorType]].join(', '), reason: errorType }}
/>
</EuiCallOut>
));
return (
<>
{content}
<EuiSpacer size={'m'} />
</>
);
};

1
x-pack/plugins/ml/public/application/explorer/reducers/explorer_reducer/reducer.ts
View file

@ -61,6 +61,7 @@ export const explorerReducer = (state: ExplorerState, nextAction: Action): Explo
seriesToPlot: payload.seriesToPlot,
// convert truthy/falsy value to Boolean
tooManyBuckets: !!payload.tooManyBuckets,
errorMessages: payload.errorMessages,
},
};
break;

79
x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_actions/results.js
View file

@ -10,15 +10,8 @@ import React, { useMemo } from 'react';
import { EuiButtonIcon, EuiToolTip } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { useCreateADLinks } from '../../../../components/custom_hooks/use_create_ad_links';
import { Link } from 'react-router-dom';
import { useMlKibana } from '../../../../contexts/kibana';
export function ResultLinks({ jobs, isManagementTable }) {
const {
services: {
http: { basePath },
},
} = useMlKibana();
export function ResultLinks({ jobs }) {
const openJobsInSingleMetricViewerText = i18n.translate(
'xpack.ml.jobsList.resultActions.openJobsInSingleMetricViewerText',
{
@ -42,6 +35,19 @@ export function ResultLinks({ jobs, isManagementTable }) {
);
const singleMetricVisible = jobs.length < 2;
const singleMetricEnabled = jobs.length === 1 && jobs[0].isSingleMetricViewerJob;
const singleMetricDisabledMessage =
jobs.length === 1 && jobs[0].isNotSingleMetricViewerJobMessage;
const singleMetricDisabledMessageText =
singleMetricDisabledMessage !== undefined
? i18n.translate('xpack.ml.jobsList.resultActions.singleMetricDisabledMessageText', {
defaultMessage: 'Disabled because {reason}.',
values: {
reason: singleMetricDisabledMessage,
},
})
: undefined;
const jobActionsDisabled = jobs.length === 1 && jobs[0].deleting === true;
const { createLinkWithUserDefaults } = useCreateADLinks();
const timeSeriesExplorerLink = useMemo(
@ -53,50 +59,29 @@ export function ResultLinks({ jobs, isManagementTable }) {
return (
<React.Fragment>
{singleMetricVisible && (
<EuiToolTip position="bottom" content={openJobsInSingleMetricViewerText}>
{isManagementTable ? (
<EuiButtonIcon
href={`${basePath.get()}/app/ml/${timeSeriesExplorerLink}`}
iconType="visLine"
aria-label={openJobsInSingleMetricViewerText}
className="results-button"
isDisabled={singleMetricEnabled === false || jobActionsDisabled === true}
data-test-subj="mlOpenJobsInSingleMetricViewerFromManagementButton"
/>
) : (
<Link to={timeSeriesExplorerLink}>
<EuiButtonIcon
iconType="visLine"
aria-label={openJobsInSingleMetricViewerText}
className="results-button"
isDisabled={singleMetricEnabled === false || jobActionsDisabled === true}
data-test-subj="mlOpenJobsInSingleMetricViewerButton"
/>
</Link>
)}
</EuiToolTip>
)}
<EuiToolTip position="bottom" content={openJobsInAnomalyExplorerText}>
{isManagementTable ? (
<EuiToolTip
position="bottom"
content={singleMetricDisabledMessageText ?? openJobsInSingleMetricViewerText}
>
<EuiButtonIcon
href={`${basePath.get()}/app/ml/${anomalyExplorerLink}`}
iconType="visTable"
href={timeSeriesExplorerLink}
iconType="visLine"
aria-label={openJobsInSingleMetricViewerText}
className="results-button"
isDisabled={singleMetricEnabled === false || jobActionsDisabled === true}
data-test-subj="mlOpenJobsInSingleMetricViewerFromManagementButton"
data-test-subj="mlOpenJobsInSingleMetricViewerButton"
/>
) : (
<Link to={anomalyExplorerLink}>
<EuiButtonIcon
iconType="visTable"
aria-label={openJobsInAnomalyExplorerText}
className="results-button"
isDisabled={jobActionsDisabled === true}
data-test-subj="mlOpenJobsInAnomalyExplorerButton"
/>
</Link>
)}
</EuiToolTip>
)}
<EuiToolTip position="bottom" content={openJobsInAnomalyExplorerText}>
<EuiButtonIcon
href={anomalyExplorerLink}
iconType="visTable"
aria-label={openJobsInAnomalyExplorerText}
className="results-button"
isDisabled={jobActionsDisabled === true}
data-test-subj="mlOpenJobsInAnomalyExplorerButton"
/>
</EuiToolTip>
<div className="actions-border" />
</React.Fragment>

2
x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list/jobs_list.js
View file

@ -237,7 +237,7 @@ export class JobsList extends Component {
name: i18n.translate('xpack.ml.jobsList.actionsLabel', {
defaultMessage: 'Actions',
}),
render: (item) => <ResultLinks jobs={[item]} isManagementTable={isManagementTable} />,
render: (item) => <ResultLinks jobs={[item]} />,
},
];

15
x-pack/plugins/ml/public/application/jobs/new_job/common/job_creator/job_creator.ts
View file

@ -33,6 +33,10 @@ import { parseInterval } from '../../../../../../common/util/parse_interval';
import { Calendar } from '../../../../../../common/types/calendars';
import { mlCalendarService } from '../../../../services/calendar_service';
import { IndexPattern } from '../../../../../../../../../src/plugins/data/public';
import {
getAggregationBucketsName,
getDatafeedAggregations,
} from '../../../../../../common/util/datafeed_utils';
export class JobCreator {
protected _type: JOB_TYPE = JOB_TYPE.SINGLE_METRIC;
@ -685,10 +689,13 @@ export class JobCreator {
}
this._aggregationFields = [];
const buckets =
this._datafeed_config.aggregations?.buckets || this._datafeed_config.aggs?.buckets;
if (buckets !== undefined) {
collectAggs(buckets, this._aggregationFields);
const aggs = getDatafeedAggregations(this._datafeed_config);
if (aggs !== undefined) {
const aggBucketsName = getAggregationBucketsName(aggs);
if (aggBucketsName !== undefined && aggs[aggBucketsName] !== undefined) {
const buckets = aggs[aggBucketsName];
collectAggs(buckets, this._aggregationFields);
}
}
}
}

32
x-pack/plugins/ml/public/application/overview/components/analytics_panel/actions.tsx
View file

@ -7,8 +7,6 @@
import React, { FC, useMemo } from 'react';
import { EuiToolTip, EuiButtonEmpty } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { Link } from 'react-router-dom';
import { useMlLink } from '../../../contexts/kibana';
import { getAnalysisType } from '../../../data_frame_analytics/common/analytics';
import { DataFrameAnalyticsListRow } from '../../../data_frame_analytics/pages/analytics_management/components/analytics_list/common';
@ -39,26 +37,24 @@ export const ViewLink: FC<Props> = ({ item }) => {
jobId: item.id,
analysisType: analysisType as DataFrameAnalysisConfigType,
},
excludeBasePath: true,
});
return (
<EuiToolTip position="bottom" content={tooltipText}>
<Link to={viewAnalyticsResultsLink}>
<EuiButtonEmpty
color="text"
size="xs"
iconType="visTable"
aria-label={viewJobResultsButtonText}
className="results-button"
data-test-subj="mlOverviewAnalyticsJobViewButton"
isDisabled={disabled}
>
{i18n.translate('xpack.ml.overview.analytics.viewActionName', {
defaultMessage: 'View',
})}
</EuiButtonEmpty>
</Link>
<EuiButtonEmpty
href={viewAnalyticsResultsLink}
color="text"
size="xs"
iconType="visTable"
aria-label={viewJobResultsButtonText}
className="results-button"
data-test-subj="mlOverviewAnalyticsJobViewButton"
isDisabled={disabled}
>
{i18n.translate('xpack.ml.overview.analytics.viewActionName', {
defaultMessage: 'View',
})}
</EuiButtonEmpty>
</EuiToolTip>
);
};

28
x-pack/plugins/ml/public/application/overview/components/anomaly_detection_panel/actions.tsx
View file

@ -7,7 +7,6 @@
import React, { FC } from 'react';
import { EuiToolTip, EuiButtonEmpty } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { Link } from 'react-router-dom';
import { MlSummaryJobs } from '../../../../../common/types/anomaly_detection_jobs';
import { useCreateADLinks } from '../../../components/custom_hooks/use_create_ad_links';
@ -27,20 +26,19 @@ export const ExplorerLink: FC<Props> = ({ jobsList }) => {
return (
<EuiToolTip position="bottom" content={openJobsInAnomalyExplorerText}>
<Link to={createLinkWithUserDefaults('explorer', jobsList)}>
<EuiButtonEmpty
color="text"
size="xs"
iconType="visTable"
aria-label={openJobsInAnomalyExplorerText}
className="results-button"
data-test-subj={`openOverviewJobsInAnomalyExplorer`}
>
{i18n.translate('xpack.ml.overview.anomalyDetection.viewActionName', {
defaultMessage: 'View',
})}
</EuiButtonEmpty>
</Link>
<EuiButtonEmpty
href={createLinkWithUserDefaults('explorer', jobsList)}
color="text"
size="xs"
iconType="visTable"
aria-label={openJobsInAnomalyExplorerText}
className="results-button"
data-test-subj={`openOverviewJobsInAnomalyExplorer`}
>
{i18n.translate('xpack.ml.overview.anomalyDetection.viewActionName', {
defaultMessage: 'View',
})}
</EuiButtonEmpty>
</EuiToolTip>
);
};

26
x-pack/plugins/ml/public/application/timeseriesexplorer/timeseriesexplorer_utils/validate_job_selection.ts
View file

@ -34,7 +34,30 @@ export function validateJobSelection(
// (e.g. if switching to this view straight from the Anomaly Explorer).
const invalidIds: string[] = difference(selectedJobIds, timeSeriesJobIds);
const validSelectedJobIds = without(selectedJobIds, ...invalidIds);
if (invalidIds.length > 0) {
// show specific reason why we can't show the single metric viewer
if (invalidIds.length === 1) {
const selectedJobId = invalidIds[0];
const selectedJob = jobsWithTimeRange.find((j) => j.id === selectedJobId);
if (selectedJob !== undefined && selectedJob.isNotSingleMetricViewerJobMessage !== undefined) {
const warningText = i18n.translate(
'xpack.ml.timeSeriesExplorer.canNotViewRequestedJobsWarningWithReasonMessage',
{
defaultMessage: `You can't view {selectedJobId} in this dashboard because {reason}.`,
values: {
selectedJobId,
reason: selectedJob.isNotSingleMetricViewerJobMessage,
},
}
);
toastNotifications.addWarning({
title: warningText,
'data-test-subj': 'mlTimeSeriesExplorerDisabledJobReasonWarningToast',
});
}
}
if (invalidIds.length > 1) {
let warningText = i18n.translate(
'xpack.ml.timeSeriesExplorer.canNotViewRequestedJobsWarningMessage',
{
@ -45,6 +68,7 @@ export function validateJobSelection(
},
}
);
if (validSelectedJobIds.length === 0 && timeSeriesJobIds.length > 0) {
warningText += i18n.translate('xpack.ml.timeSeriesExplorer.autoSelectingFirstJobText', {
defaultMessage: ', auto selecting first job',

14
x-pack/plugins/ml/server/models/job_service/jobs.ts
View file

@ -8,7 +8,10 @@ import { i18n } from '@kbn/i18n';
import { uniq } from 'lodash';
import Boom from '@hapi/boom';
import { IScopedClusterClient } from 'kibana/server';
import { parseTimeIntervalForJob } from '../../../common/util/job_utils';
import {
getSingleMetricViewerJobErrorMessage,
parseTimeIntervalForJob,
} from '../../../common/util/job_utils';
import { JOB_STATE, DATAFEED_STATE } from '../../../common/constants/states';
import {
MlSummaryJob,
@ -27,7 +30,6 @@ import { fillResultsWithTimeouts, isRequestTimeout } from './error_utils';
import {
getEarliestDatafeedStartTime,
getLatestDataOrBucketTimestamp,
isTimeSeriesViewJob,
} from '../../../common/util/job_utils';
import { groupsProvider } from './groups';
import type { MlClient } from '../../lib/ml_client';
@ -175,6 +177,7 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
const hasDatafeed =
typeof job.datafeed_config === 'object' && Object.keys(job.datafeed_config).length > 0;
const dataCounts = job.data_counts;
const errorMessage = getSingleMetricViewerJobErrorMessage(job);
const tempJob: MlSummaryJob = {
id: job.job_id,
@ -200,7 +203,8 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
dataCounts?.latest_record_timestamp,
dataCounts?.latest_bucket_timestamp
),
isSingleMetricViewerJob: isTimeSeriesViewJob(job),
isSingleMetricViewerJob: errorMessage === undefined,
isNotSingleMetricViewerJobMessage: errorMessage,
nodeName: job.node ? job.node.name : undefined,
deleting: job.deleting || undefined,
};
@ -242,13 +246,15 @@ export function jobsProvider(client: IScopedClusterClient, mlClient: MlClient) {
);
timeRange.from = dataCounts.earliest_record_timestamp;
}
const errorMessage = getSingleMetricViewerJobErrorMessage(job);
const tempJob = {
id: job.job_id,
job_id: job.job_id,
groups: Array.isArray(job.groups) ? job.groups.sort() : [],
isRunning: hasDatafeed && job.datafeed_config.state === 'started',
isSingleMetricViewerJob: isTimeSeriesViewJob(job),
isSingleMetricViewerJob: errorMessage === undefined,
isNotSingleMetricViewerJobMessage: errorMessage,
timeRange,
};

474
x-pack/test/functional/apps/ml/anomaly_detection/aggregated_scripted_job.ts Normal file
View file

@ -0,0 +1,474 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { FtrProviderContext } from '../../../ftr_provider_context';
import { Datafeed, Job } from '../../../../../plugins/ml/common/types/anomaly_detection_jobs';
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const ml = getService('ml');
const ts = Date.now();
const supportedTestSuites = [
{
suiteTitle: 'supported job with aggregation field',
jobConfig: {
job_id: `fq_supported_aggs_${ts}`,
job_type: 'anomaly_detector',
description: '',
analysis_config: {
bucket_span: '30m',
summary_count_field_name: 'doc_count',
detectors: [
{
function: 'mean',
field_name: 'responsetime_avg',
detector_description: 'mean(responsetime_avg)',
},
],
influencers: ['airline'],
},
analysis_limits: {
model_memory_limit: '11MB',
},
data_description: {
time_field: '@timestamp',
time_format: 'epoch_ms',
},
model_plot_config: {
enabled: false,
annotations_enabled: false,
},
model_snapshot_retention_days: 1,
results_index_name: 'shared',
allow_lazy_open: false,
groups: [],
} as Job,
datafeedConfig: ({
datafeed_id: `datafeed-fq_supported_aggs_${ts}`,
job_id: `fq_supported_aggs_${ts}`,
chunking_config: {
mode: 'manual',
time_span: '900000000ms',
},
indices_options: {
expand_wildcards: ['open'],
ignore_unavailable: false,
allow_no_indices: true,
ignore_throttled: true,
},
query: {
match_all: {},
},
indices: ['ft_farequote'],
aggregations: {
buckets: {
date_histogram: {
field: '@timestamp',
fixed_interval: '15m',
},
aggregations: {
'@timestamp': {
max: {
field: '@timestamp',
},
},
airline: {
terms: {
field: 'airline',
size: 100,
},
aggregations: {
responsetime_avg: {
avg: {
field: 'responsetime',
},
},
},
},
},
},
},
scroll_size: 1000,
delayed_data_check_config: {
enabled: true,
},
} as unknown) as Datafeed,
},
{
suiteTitle: 'supported job with scripted field',
jobConfig: {
job_id: `fq_supported_script_${ts}`,
job_type: 'anomaly_detector',
description: '',
analysis_config: {
bucket_span: '15m',
detectors: [
{
function: 'mean',
field_name: 'actual_taxed',
detector_description: 'mean(actual_taxed) by gender_currency',
},
],
influencers: [],
},
analysis_limits: {
model_memory_limit: '11MB',
},
data_description: {
time_field: 'order_date',
time_format: 'epoch_ms',
},
model_plot_config: {
enabled: true,
annotations_enabled: false,
},
model_snapshot_retention_days: 10,
daily_model_snapshot_retention_after_days: 1,
results_index_name: 'shared',
allow_lazy_open: false,
groups: [],
} as Job,
datafeedConfig: ({
chunking_config: {
mode: 'auto',
},
indices_options: {
expand_wildcards: ['open'],
ignore_unavailable: false,
allow_no_indices: true,
ignore_throttled: true,
},
query: {
bool: {
must: [
{
match_all: {},
},
],
},
},
indices: ['ft_ecommerce'],
script_fields: {
actual_taxed: {
script: {
source: "doc['taxful_total_price'].value * 1.825",
lang: 'painless',
},
ignore_failure: false,
},
},
scroll_size: 1000,
delayed_data_check_config: {
enabled: true,
},
job_id: `fq_supported_script_${ts}`,
datafeed_id: `datafeed-fq_supported_script_${ts}`,
} as unknown) as Datafeed,
},
];
const unsupportedTestSuites = [
{
suiteTitle: 'unsupported job with bucket_script aggregation field',
jobConfig: {
job_id: `fq_unsupported_aggs_${ts}`,
job_type: 'anomaly_detector',
description: '',
analysis_config: {
bucket_span: '15m',
summary_count_field_name: 'doc_count',
detectors: [
{
function: 'mean',
field_name: 'max_delta',
partition_field_name: 'airlines',
detector_description: 'mean(max_delta) partition airline',
},
],
influencers: ['airlines'],
},
analysis_limits: {
model_memory_limit: '11MB',
},
data_description: {
time_field: '@timestamp',
time_format: 'epoch_ms',
},
model_plot_config: {
enabled: false,
annotations_enabled: false,
},
model_snapshot_retention_days: 1,
results_index_name: 'shared',
allow_lazy_open: false,
groups: [],
} as Job,
datafeedConfig: ({
datafeed_id: `datafeed-fq_unsupported_aggs_${ts}`,
job_id: `fq_unsupported_aggs_${ts}`,
chunking_config: {
mode: 'manual',
time_span: '900000000ms',
},
indices_options: {
expand_wildcards: ['open'],
ignore_unavailable: false,
allow_no_indices: true,
ignore_throttled: true,
},
query: {
match_all: {},
},
indices: ['ft_farequote'],
aggregations: {
buckets: {
date_histogram: {
field: '@timestamp',
fixed_interval: '15m',
time_zone: 'UTC',
},
aggregations: {
'@timestamp': {
max: {
field: '@timestamp',
},
},
airlines: {
terms: {
field: 'airline',
size: 200,
order: {
_count: 'desc',
},
},
aggregations: {
max: {
max: {
field: 'responsetime',
},
},
min: {
min: {
field: 'responsetime',
},
},
max_delta: {
bucket_script: {
buckets_path: {
maxval: 'max',
minval: 'min',
},
script: 'params.maxval - params.minval',
},
},
},
},
},
},
},
scroll_size: 1000,
delayed_data_check_config: {
enabled: true,
},
} as unknown) as Datafeed,
},
{
suiteTitle: 'unsupported job with partition by of a scripted field',
jobConfig: {
job_id: `fq_unsupported_script_${ts}`,
job_type: 'anomaly_detector',
description: '',
analysis_config: {
bucket_span: '15m',
detectors: [
{
function: 'mean',
field_name: 'actual_taxed',
by_field_name: 'gender_currency',
detector_description: 'mean(actual_taxed) by gender_currency',
},
],
influencers: ['gender_currency'],
},
analysis_limits: {
model_memory_limit: '11MB',
},
data_description: {
time_field: 'order_date',
time_format: 'epoch_ms',
},
model_plot_config: {
enabled: false,
annotations_enabled: false,
},
model_snapshot_retention_days: 10,
daily_model_snapshot_retention_after_days: 1,
results_index_name: 'shared',
allow_lazy_open: false,
groups: [],
} as Job,
datafeedConfig: ({
chunking_config: {
mode: 'auto',
},
indices_options: {
expand_wildcards: ['open'],
ignore_unavailable: false,
allow_no_indices: true,
ignore_throttled: true,
},
query: {
bool: {
must: [
{
match_all: {},
},
],
},
},
indices: ['ft_ecommerce'],
script_fields: {
actual_taxed: {
script: {
source: "doc['taxful_total_price'].value * 1.825",
lang: 'painless',
},
ignore_failure: false,
},
gender_currency: {
script: {
source: "doc['customer_gender'].value + '_' + doc['currency'].value",
lang: 'painless',
},
ignore_failure: false,
},
},
scroll_size: 1000,
delayed_data_check_config: {
enabled: true,
},
job_id: `fq_unsupported_script_${ts}`,
datafeed_id: `datafeed-fq_unsupported_script_${ts}`,
} as unknown) as Datafeed,
},
];
describe('aggregated or scripted job', function () {
this.tags(['mlqa']);
before(async () => {
await esArchiver.loadIfNeeded('ml/farequote');
await esArchiver.loadIfNeeded('ml/ecommerce');
await ml.testResources.createIndexPatternIfNeeded('ft_farequote', '@timestamp');
await ml.testResources.createIndexPatternIfNeeded('ft_ecommerce', 'order_date');
await ml.testResources.setKibanaTimeZoneToUTC();
await ml.securityUI.loginAsMlPowerUser();
});
after(async () => {
await ml.api.cleanMlIndices();
});
for (const testData of supportedTestSuites) {
describe(testData.suiteTitle, function () {
before(async () => {
await ml.api.createAndRunAnomalyDetectionLookbackJob(
testData.jobConfig,
testData.datafeedConfig
);
});
it('opens a job from job list link', async () => {
await ml.testExecution.logTestStep('navigate to job list');
await ml.navigation.navigateToMl();
await ml.navigation.navigateToJobManagement();
await ml.testExecution.logTestStep(
'check that the single metric viewer button is enabled'
);
await ml.jobTable.waitForJobsToLoad();
await ml.jobTable.filterWithSearchString(testData.jobConfig.job_id, 1);
await ml.jobTable.assertJobActionSingleMetricViewerButtonEnabled(
testData.jobConfig.job_id,
true
);
await ml.testExecution.logTestStep('opens job in single metric viewer');
await ml.jobTable.clickOpenJobInSingleMetricViewerButton(testData.jobConfig.job_id);
await ml.commonUI.waitForMlLoadingIndicatorToDisappear();
});
it('displays job results correctly in both anomaly explorer and single metric viewer', async () => {
await ml.testExecution.logTestStep('should display the chart');
await ml.singleMetricViewer.assertChartExist();
await ml.testExecution.logTestStep('should navigate to anomaly explorer');
await ml.navigation.navigateToAnomalyExplorerViaSingleMetricViewer();
await ml.testExecution.logTestStep('pre-fills the job selection');
await ml.jobSelection.assertJobSelection([testData.jobConfig.job_id]);
await ml.testExecution.logTestStep('displays the swimlanes');
await ml.anomalyExplorer.assertOverallSwimlaneExists();
await ml.anomalyExplorer.assertSwimlaneViewByExists();
});
});
}
for (const testData of unsupportedTestSuites) {
describe(testData.suiteTitle, function () {
before(async () => {
await ml.api.createAndRunAnomalyDetectionLookbackJob(
testData.jobConfig,
testData.datafeedConfig
);
});
it('opens a job from job list link', async () => {
await ml.testExecution.logTestStep('navigate to job list');
await ml.navigation.navigateToMl();
await ml.navigation.navigateToJobManagement();
await ml.testExecution.logTestStep(
'check that the single metric viewer button is disabled'
);
await ml.jobTable.waitForJobsToLoad();
await ml.jobTable.filterWithSearchString(testData.jobConfig.job_id, 1);
await ml.jobTable.assertJobActionSingleMetricViewerButtonEnabled(
testData.jobConfig.job_id,
false
);
await ml.testExecution.logTestStep('open job in anomaly explorer');
await ml.jobTable.clickOpenJobInAnomalyExplorerButton(testData.jobConfig.job_id);
await ml.commonUI.waitForMlLoadingIndicatorToDisappear();
});
it('displays job results', async () => {
await ml.testExecution.logTestStep('pre-fills the job selection');
await ml.jobSelection.assertJobSelection([testData.jobConfig.job_id]);
await ml.testExecution.logTestStep('displays the swimlanes');
await ml.anomalyExplorer.assertOverallSwimlaneExists();
await ml.anomalyExplorer.assertSwimlaneViewByExists();
// TODO: click on swimlane cells to trigger warning callouts
// when we figure out a way to click inside canvas renderings
await ml.testExecution.logTestStep('should navigate to single metric viewer');
await ml.navigation.navigateToSingleMetricViewerViaAnomalyExplorer();
await ml.testExecution.logTestStep(
'should show warning message and redirect single metric viewer to another job'
);
await ml.singleMetricViewer.assertDisabledJobReasonWarningToastExist();
await ml.jobSelection.assertJobSelectionNotContains(testData.jobConfig.job_id);
});
});
}
});
}

1
x-pack/test/functional/apps/ml/anomaly_detection/index.ts
View file

@ -20,5 +20,6 @@ export default function ({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./categorization_job'));
loadTestFile(require.resolve('./date_nanos_job'));
loadTestFile(require.resolve('./annotations'));
loadTestFile(require.resolve('./aggregated_scripted_job'));
});
}

13
x-pack/test/functional/services/ml/job_selection.ts
View file

@ -23,5 +23,18 @@ export function MachineLearningJobSelectionProvider({ getService }: FtrProviderC
`Job selection should display jobs or groups '${jobOrGroupIds}' (got '${actualJobOrGroupLabels}')`
);
},
async assertJobSelectionNotContains(jobOrGroupId: string) {
const selectedJobsOrGroups = await testSubjects.findAll(
'mlJobSelectionBadges > ~mlJobSelectionBadge'
);
const actualJobOrGroupLabels = await Promise.all(
selectedJobsOrGroups.map(async (badge) => await badge.getVisibleText())
);
expect(actualJobOrGroupLabels).to.not.contain(
jobOrGroupId,
`Job selection should not contain job or group '${jobOrGroupId}' (got '${actualJobOrGroupLabels}')`
);
},
};
}

8
x-pack/test/functional/services/ml/single_metric_viewer.ts
View file

@ -187,5 +187,13 @@ export function MachineLearningSingleMetricViewerProvider(
);
await this.assertEntityConfig(entityFieldName, anomalousOnly, sortBy, order);
},
async assertToastMessageExists(dataTestSubj: string) {
const toast = await testSubjects.find(dataTestSubj);
expect(toast).not.to.be(undefined);
},
async assertDisabledJobReasonWarningToastExist() {
await this.assertToastMessageExists('mlTimeSeriesExplorerDisabledJobReasonWarningToast');
},
};
}