From 1365799a0698d4fd82261ce2cc8f61f1380f3144 Mon Sep 17 00:00:00 2001 From: Pete Harverson Date: Thu, 24 May 2018 15:49:00 +0100 Subject: [PATCH] [ML] Convert anomalies table to EUI / React (#19352) * [ML] Convert anomalies table to EUI / React * [ML] Edits to EUI / React anomalies table after review --- .../anomalies_table/anomalies_table.html | 12 - .../anomalies_table/anomalies_table.js | 394 +++++++ .../anomalies_table_directive.js | 966 +----------------- .../anomalies_table_links.html | 20 - .../anomalies_table_service.js | 18 +- .../anomalies_table/anomaly_details.js | 326 ++++++ .../anomalies_table/description_cell.js | 53 + .../components/anomalies_table/entity_cell.js | 56 + .../expanded_row/expanded_row.html | 108 -- .../expanded_row/expanded_row_directive.js | 217 ---- .../components/anomalies_table/index.js | 5 +- .../influencers_cell.js | 72 +- .../influencers_cell_directive.js | 18 - .../components/anomalies_table/links_menu.js | 414 ++++++++ .../anomalies_table/styles/main.less | 234 ++--- .../plugins/ml/public/explorer/explorer.html | 11 +- .../ml/public/explorer/explorer_controller.js | 168 ++- .../ml/public/explorer/styles/main.less | 3 +- .../public/services/ml_api_service/results.js | 2 + .../timeseries_chart_directive.js | 2 +- .../timeseriesexplorer.html | 7 +- .../timeseriesexplorer_controller.js | 95 +- .../timeseriesexplorer_utils.js | 8 +- .../build_anomaly_table_items.js | 32 +- .../models/results_service/results_service.js | 17 +- .../ml/server/routes/results_service.js | 2 + 26 files changed, 1675 insertions(+), 1585 deletions(-) delete mode 100644 x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.html create mode 100644 x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.js delete mode 100644 x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_links.html create mode 100644 x-pack/plugins/ml/public/components/anomalies_table/anomaly_details.js create mode 100644 x-pack/plugins/ml/public/components/anomalies_table/description_cell.js create mode 100644 x-pack/plugins/ml/public/components/anomalies_table/entity_cell.js delete mode 100644 x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row.html delete mode 100644 x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js rename x-pack/plugins/ml/public/components/anomalies_table/{influencers_cell => }/influencers_cell.js (57%) delete mode 100644 x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell_directive.js create mode 100644 x-pack/plugins/ml/public/components/anomalies_table/links_menu.js diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.html b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.html deleted file mode 100644 index 913fbbab7513..000000000000 --- a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.html +++ /dev/null @@ -1,12 +0,0 @@ -
-
-

No matching results found

-
- - - - -
diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.js b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.js new file mode 100644 index 000000000000..ef28da7ae231 --- /dev/null +++ b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table.js @@ -0,0 +1,394 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +/* + * React table for displaying a list of anomalies. + */ + +import PropTypes from 'prop-types'; +import _ from 'lodash'; +import $ from 'jquery'; + +import React, { + Component +} from 'react'; + +import { + EuiButtonIcon, + EuiFlexGroup, + EuiFlexItem, + EuiHealth, + EuiInMemoryTable, + EuiText +} from '@elastic/eui'; + +import { formatDate } from '@elastic/eui/lib/services/format'; + +import { DescriptionCell } from './description_cell'; +import { EntityCell } from './entity_cell'; +import { InfluencersCell } from './influencers_cell'; +import { AnomalyDetails } from './anomaly_details'; +import { LinksMenu } from './links_menu'; + +import { mlAnomaliesTableService } from './anomalies_table_service'; +import { mlFieldFormatService } from 'plugins/ml/services/field_format_service'; +import { getSeverityColor } from 'plugins/ml/../common/util/anomaly_utils'; +import { formatValue } from 'plugins/ml/formatters/format_value'; + + +const INFLUENCERS_LIMIT = 5; // Maximum number of influencers to display before a 'show more' link is added. + + +function renderTime(date, aggregationInterval) { + if (aggregationInterval === 'hour') { + return formatDate(date, 'MMMM Do YYYY, HH:mm'); + } else if (aggregationInterval === 'second') { + return formatDate(date, 'MMMM Do YYYY, HH:mm:ss'); + } else { + return formatDate(date, 'MMMM Do YYYY'); + } +} + +function showLinksMenuForItem(item) { + return item.isTimeSeriesViewDetector || + item.entityName === 'mlcategory' || + item.customUrls !== undefined; +} + +function getColumns( + items, + examplesByJobId, + isAggregatedData, + interval, + timefilter, + showViewSeriesLink, + itemIdToExpandedRowMap, + toggleRow, + filter) { + const columns = [ + { + name: '', + render: (item) => ( + toggleRow(item)} + iconType={itemIdToExpandedRowMap[item.rowId] ? 'arrowDown' : 'arrowRight'} + aria-label={itemIdToExpandedRowMap[item.rowId] ? 'Hide details' : 'Show details'} + data-row-id={item.rowId} + /> + ) + }, + { + field: 'time', + name: 'time', + dataType: 'date', + render: (date) => renderTime(date, interval), + sortable: true + }, + { + field: 'severity', + name: `${(isAggregatedData === true) ? 'max ' : ''}severity`, + render: (score) => ( + + {score >= 1 ? Math.floor(score) : '< 1'} + + ), + sortable: true + }, + { + field: 'detector', + name: 'detector', + sortable: true + } + ]; + + if (items.some(item => item.entityValue !== undefined)) { + columns.push({ + field: 'entityValue', + name: 'found for', + render: (entityValue, item) => ( + + ), + sortable: true + }); + } + + if (items.some(item => item.influencers !== undefined)) { + columns.push({ + field: 'influencers', + name: 'influenced by', + render: (influencers) => ( + + ), + sortable: true + }); + } + + // Map the additional 'sort' fields to the actual, typical and description + // fields to ensure sorting is done correctly on the underlying metric value + // and not on e.g. the actual values array as a String. + if (items.some(item => item.actual !== undefined)) { + columns.push({ + field: 'actualSort', + name: 'actual', + render: (actual, item) => { + const fieldFormat = mlFieldFormatService.getFieldFormat(item.jobId, item.source.detector_index); + return formatValue(item.actual, item.source.function, fieldFormat); + }, + sortable: true + }); + } + + if (items.some(item => item.typical !== undefined)) { + columns.push({ + field: 'typicalSort', + name: 'typical', + render: (typical, item) => { + const fieldFormat = mlFieldFormatService.getFieldFormat(item.jobId, item.source.detector_index); + return formatValue(item.typical, item.source.function, fieldFormat); + }, + sortable: true + }); + + // Assume that if we are showing typical, there will be an actual too, + // so we can add a column to describe how actual compares to typical. + const nonTimeOfDayOrWeek = items.some((item) => { + const summaryRecFunc = item.source.function; + return summaryRecFunc !== 'time_of_day' && summaryRecFunc !== 'time_of_week'; + }); + if (nonTimeOfDayOrWeek === true) { + columns.push({ + field: 'metricDescriptionSort', + name: 'description', + render: (metricDescriptionSort, item) => ( + + ), + sortable: true + }); + } + } + + columns.push({ + field: 'jobId', + name: 'job ID', + sortable: true + }); + + const showExamples = items.some(item => item.entityName === 'mlcategory'); + const showLinks = (showViewSeriesLink === true) || items.some(item => showLinksMenuForItem(item)); + + if (showLinks === true) { + columns.push({ + name: 'links', + render: (item) => { + if (showLinksMenuForItem(item) === true) { + return ( + + ); + } else { + return null; + } + }, + sortable: false + }); + } + + if (showExamples === true) { + columns.push({ + name: 'category examples', + sortable: false, + truncateText: true, + render: (item) => { + const examples = _.get(examplesByJobId, [item.jobId, item.entityValue], []); + return ( + + {examples.map((example, i) => { + return {example}; + } + )} + + ); + } + }); + } + + return columns; +} + +class AnomaliesTable extends Component { + constructor(props) { + super(props); + + this.state = { + itemIdToExpandedRowMap: {} + }; + } + + isShowingAggregatedData = () => { + return (this.props.tableData.interval !== 'second'); + }; + + static getDerivedStateFromProps(nextProps, prevState) { + // Update the itemIdToExpandedRowMap state if a change to the table data has resulted + // in an anomaly that was previously expanded no longer being in the data. + const itemIdToExpandedRowMap = prevState.itemIdToExpandedRowMap; + const prevExpandedNotInData = Object.keys(itemIdToExpandedRowMap).find((rowId) => { + const matching = nextProps.tableData.anomalies.find((anomaly) => { + return anomaly.rowId === rowId; + }); + + return (matching === undefined); + }); + + if (prevExpandedNotInData !== undefined) { + // Anomaly data has changed and an anomaly that was previously expanded is no longer in the data. + return { + itemIdToExpandedRowMap: {} + }; + } + + // Return null to indicate no change to state. + return null; + } + + toggleRow = (item) => { + const itemIdToExpandedRowMap = { ...this.state.itemIdToExpandedRowMap }; + if (itemIdToExpandedRowMap[item.rowId]) { + delete itemIdToExpandedRowMap[item.rowId]; + } else { + const examples = (item.entityName === 'mlcategory') ? + _.get(this.props.tableData, ['examplesByJobId', item.jobId, item.entityValue]) : undefined; + itemIdToExpandedRowMap[item.rowId] = ( + + ); + } + this.setState({ itemIdToExpandedRowMap }); + }; + + onMouseOver = (event) => { + // Triggered when the mouse is somewhere over the table. + // Traverse through the table DOM to find the expand/collapse + // button which stores the ID of the row. + let mouseOverRecord = undefined; + const target = $(event.target); + const parentRow = target.closest('tr'); + const firstCell = parentRow.children('td').first(); + if (firstCell !== undefined) { + const expandButton = firstCell.find('button').first(); + if (expandButton.length > 0) { + const rowId = expandButton.attr('data-row-id'); + mouseOverRecord = this.props.tableData.anomalies.find((anomaly) => { + return (anomaly.rowId === rowId); + }); + } + } + + if (this.mouseOverRecord !== undefined) { + if (mouseOverRecord === undefined || this.mouseOverRecord.rowId !== mouseOverRecord.rowId) { + // Mouse is over a different row, fire mouseleave on the previous record. + mlAnomaliesTableService.anomalyRecordMouseleave.changed(this.mouseOverRecord); + + if (mouseOverRecord !== undefined) { + // Mouse is over a new row, fire mouseenter on the new record. + mlAnomaliesTableService.anomalyRecordMouseenter.changed(mouseOverRecord); + } + } + } else if (mouseOverRecord !== undefined) { + // Mouse is now over a row, fire mouseenter on the record. + mlAnomaliesTableService.anomalyRecordMouseenter.changed(mouseOverRecord); + } + + this.mouseOverRecord = mouseOverRecord; + }; + + onMouseLeave = () => { + if (this.mouseOverRecord !== undefined) { + mlAnomaliesTableService.anomalyRecordMouseleave.changed(this.mouseOverRecord); + } + }; + + render() { + const { timefilter, tableData, filter } = this.props; + + if (tableData === undefined || + tableData.anomalies === undefined || tableData.anomalies.length === 0) { + return ( + + + +

No matching anomalies found

+
+
+
+ ); + } + + const columns = getColumns( + tableData.anomalies, + tableData.examplesByJobId, + this.isShowingAggregatedData(), + tableData.interval, + timefilter, + tableData.showViewSeriesLink, + this.state.itemIdToExpandedRowMap, + this.toggleRow, + filter); + + const sorting = { + sort: { + field: 'severity', + direction: 'desc', + } + }; + + return ( + + ); + } +} +AnomaliesTable.propTypes = { + timefilter: PropTypes.object.isRequired, + tableData: PropTypes.object, + filter: PropTypes.func +}; + +export { AnomaliesTable }; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js index da5f64fb5a96..3cfd962602a3 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_directive.js @@ -5,954 +5,26 @@ */ +import 'ngreact'; -/* - * AngularJS directive for rendering a table of Machine Learning anomalies. - */ - -import moment from 'moment'; -import _ from 'lodash'; -import rison from 'rison-node'; - -import { notify } from 'ui/notify'; -import { ES_FIELD_TYPES } from 'plugins/ml/../common/constants/field_types'; -import { parseInterval } from 'plugins/ml/../common/util/parse_interval'; -import { formatValue } from 'plugins/ml/formatters/format_value'; -import { getUrlForRecord } from 'plugins/ml/util/custom_url_utils'; -import { replaceStringTokens, mlEscape } from 'plugins/ml/util/string_utils'; -import { isTimeSeriesViewDetector } from 'plugins/ml/../common/util/job_utils'; -import { getIndexPatternIdFromName } from 'plugins/ml/util/index_utils'; -import { - getEntityFieldName, - getEntityFieldValue, - showActualForFunction, - showTypicalForFunction, - getSeverity -} from 'plugins/ml/../common/util/anomaly_utils'; -import { getFieldTypeFromMapping } from 'plugins/ml/services/mapping_service'; -import { ml } from 'plugins/ml/services/ml_api_service'; -import { mlJobService } from 'plugins/ml/services/job_service'; -import { mlFieldFormatService } from 'plugins/ml/services/field_format_service'; -import template from './anomalies_table.html'; - -import 'plugins/ml/components/controls'; -import 'plugins/ml/components/paginated_table'; -import 'plugins/ml/formatters/metric_change_description'; -import './expanded_row/expanded_row_directive'; -import './influencers_cell/influencers_cell_directive'; - -import linkControlsHtml from './anomalies_table_links.html'; -import chrome from 'ui/chrome'; -import openRowArrow from 'plugins/ml/components/paginated_table/open.html'; import { uiModules } from 'ui/modules'; -const module = uiModules.get('apps/ml'); - -module.directive('mlAnomaliesTable', function ( - $window, - $route, - timefilter, - Private, - mlAnomaliesTableService, - mlSelectIntervalService, - mlSelectSeverityService) { - - return { - restrict: 'E', - scope: { - anomalyRecords: '=', - timeFieldName: '=', - showViewSeriesLink: '=', - filteringEnabled: '=' - }, - template, - link: function (scope, element) { - // Previously, we instantiated a new AppState here for the - // severity threshold and interval setting, thus resetting it on every - // reload. Now that this is handled differently via services and them - // being singletons, we need to explicitly reset the setting's state, - // otherwise the state would be retained across multiple instances of - // these settings. Should we want to change this behavior, e.g. to - // store the setting of the severity threshold across pages, we can - // just remove these resets. - mlSelectIntervalService.state.reset().changed(); - mlSelectSeverityService.state.reset().changed(); - - scope.momentInterval = 'second'; - - scope.table = {}; - scope.table.perPage = 25; - scope.table.columns = []; - scope.table.rows = []; - scope.rowScopes = []; - - scope.influencersLimit = 5; - - scope.categoryExamplesByJob = {}; - const MAX_NUMBER_CATEGORY_EXAMPLES = 10; // Max number of examples to show in table cell or expanded row (engine default is to store 4). - - mlSelectIntervalService.state.watch(updateTableData); - mlSelectSeverityService.state.watch(updateTableData); - - scope.$watchCollection('anomalyRecords', updateTableData); - - element.on('$destroy', () => { - mlSelectIntervalService.state.unwatch(updateTableData); - mlSelectSeverityService.state.unwatch(updateTableData); - scope.$destroy(); - }); - - scope.isShowingAggregatedData = function () { - const interval = mlSelectIntervalService.state.get('interval'); - return (interval.display !== 'Show all'); - }; - - scope.getExamplesForCategory = function (jobId, categoryId) { - return _.get(scope.categoryExamplesByJob, [jobId, categoryId], []); - }; - - scope.viewSeries = function (record) { - const bounds = timefilter.getActiveBounds(); - const from = bounds.min.toISOString(); // e.g. 2016-02-08T16:00:00.000Z - const to = bounds.max.toISOString(); - - // Zoom to show 50 buckets either side of the record. - const recordTime = moment(record[scope.timeFieldName]); - const zoomFrom = recordTime.subtract(50 * record.bucket_span, 's').toISOString(); - const zoomTo = recordTime.add(100 * record.bucket_span, 's').toISOString(); - - // Extract the by, over and partition fields for the record. - const entityCondition = {}; - - if (_.has(record, 'partition_field_value')) { - entityCondition[record.partition_field_name] = record.partition_field_value; - } - - if (_.has(record, 'over_field_value')) { - entityCondition[record.over_field_name] = record.over_field_value; - } - - if (_.has(record, 'by_field_value')) { - // Note that analyses with by and over fields, will have a top-level by_field_name, - // but the by_field_value(s) will be in the nested causes array. - // TODO - drilldown from cause in expanded row only? - entityCondition[record.by_field_name] = record.by_field_value; - } - - // Use rison to build the URL . - const _g = rison.encode({ - ml: { - jobIds: [record.job_id] - }, - refreshInterval: { - display: 'Off', - pause: false, - value: 0 - }, - time: { - from: from, - to: to, - mode: 'absolute' - } - }); - - const _a = rison.encode({ - mlTimeSeriesExplorer: { - zoom: { - from: zoomFrom, - to: zoomTo - }, - detectorIndex: record.detector_index, - entities: entityCondition, - }, - filters: [], - query: { - query_string: { - analyze_wildcard: true, - query: '*' - } - } - }); - - // Need to encode the _a parameter in case any entities contain unsafe characters such as '+'. - let path = chrome.getBasePath(); - path += '/app/ml#/timeseriesexplorer'; - path += '?_g=' + _g; - path += '&_a=' + encodeURIComponent(_a); - $window.open(path, '_blank'); - }; - - scope.viewExamples = function (record) { - const categoryId = getEntityFieldValue(record); - const job = mlJobService.getJob(record.job_id); - const categorizationFieldName = job.analysis_config.categorization_field_name; - const datafeedIndices = job.datafeed_config.indices; - // Find the type of the categorization field i.e. text (preferred) or keyword. - // Uses the first matching field found in the list of indices in the datafeed_config. - // attempt to load the field type using each index. we have to do it this way as _field_caps - // doesn't specify which index a field came from unless there is a clash. - let i = 0; - findFieldType(datafeedIndices[i]); - - function findFieldType(index) { - getFieldTypeFromMapping(index, categorizationFieldName) - .then((resp) => { - if (resp !== '') { - createAndOpenUrl(index, resp); - } else { - i++; - if (i < datafeedIndices.length) { - findFieldType(datafeedIndices[i]); - } else { - error(); - } - } - }) - .catch(() => { - error(); - }); - } - - function createAndOpenUrl(index, categorizationFieldType) { - // Find the ID of the index pattern with a title attribute which matches the - // index configured in the datafeed. If a Kibana index pattern has not been created - // for this index, then the user will see a warning message on the Discover tab advising - // them that no matching index pattern has been configured. - const indexPatternId = getIndexPatternIdFromName(index); - - // Get the definition of the category and use the terms or regex to view the - // matching events in the Kibana Discover tab depending on whether the - // categorization field is of mapping type text (preferred) or keyword. - ml.results.getCategoryDefinition(record.job_id, categoryId) - .then((resp) => { - let query = null; - // Build query using categorization regex (if keyword type) or terms (if text type). - // Check for terms or regex in case categoryId represents an anomaly from the absence of the - // categorization field in documents (usually indicated by a categoryId of -1). - if (categorizationFieldType === ES_FIELD_TYPES.KEYWORD) { - if (resp.regex) { - query = `${categorizationFieldName}:/${resp.regex}/`; - } - } else { - if (resp.terms) { - query = `${categorizationFieldName}:` + resp.terms.split(' ').join(` AND ${categorizationFieldName}:`); - } - } - - const recordTime = moment(record[scope.timeFieldName]); - const from = recordTime.toISOString(); - const to = recordTime.add(record.bucket_span, 's').toISOString(); - - // Use rison to build the URL . - const _g = rison.encode({ - refreshInterval: { - display: 'Off', - pause: false, - value: 0 - }, - time: { - from: from, - to: to, - mode: 'absolute' - } - }); - - const appStateProps = { - index: indexPatternId, - filters: [] - }; - if (query !== null) { - appStateProps.query = { - query_string: { - analyze_wildcard: true, - query: query - } - }; - } - const _a = rison.encode(appStateProps); - - // Need to encode the _a parameter as it will contain characters such as '+' if using the regex. - let path = chrome.getBasePath(); - path += '/app/kibana#/discover'; - path += '?_g=' + _g; - path += '&_a=' + encodeURIComponent(_a); - $window.open(path, '_blank'); - - }).catch((resp) => { - console.log('viewExamples(): error loading categoryDefinition:', resp); - }); - - } - - function error() { - console.log(`viewExamples(): error finding type of field ${categorizationFieldName} in indices:`, - datafeedIndices); - notify.error(`Unable to view examples of documents with mlcategory ${categoryId} ` + - `as no mapping could be found for the categorization field ${categorizationFieldName}`, - { lifetime: 30000 }); - } - }; - - scope.openCustomUrl = function (customUrl, record) { - console.log('Anomalies Table - open customUrl for record:', customUrl, record); - - // If url_value contains $earliest$ and $latest$ tokens, add in times to the source record. - const timestamp = record[scope.timeFieldName]; - const configuredUrlValue = customUrl.url_value; - const timeRangeInterval = parseInterval(customUrl.time_range); - if (configuredUrlValue.includes('$earliest$')) { - let earliestMoment = moment(timestamp); - if (timeRangeInterval !== null) { - earliestMoment.subtract(timeRangeInterval); - } else { - earliestMoment = moment(timestamp).startOf(scope.momentInterval); - if (scope.momentInterval === 'hour') { - // Start from the previous hour. - earliestMoment.subtract(1, 'h'); - } - } - record.earliest = earliestMoment.toISOString(); // e.g. 2016-02-08T16:00:00.000Z - } - - if (configuredUrlValue.includes('$latest$')) { - let latestMoment = moment(timestamp).add(record.bucket_span, 's'); - if (timeRangeInterval !== null) { - latestMoment.add(timeRangeInterval); - } else { - if (scope.isShowingAggregatedData()) { - latestMoment = moment(timestamp).endOf(scope.momentInterval); - if (scope.momentInterval === 'hour') { - // Show to the end of the next hour. - latestMoment.add(1, 'h'); // e.g. 2016-02-08T18:59:59.999Z - } - } - } - record.latest = latestMoment.toISOString(); - } - - // If url_value contains $mlcategoryterms$ or $mlcategoryregex$, add in the - // terms and regex for the selected categoryId to the source record. - if ((configuredUrlValue.includes('$mlcategoryterms$') || configuredUrlValue.includes('$mlcategoryregex$')) - && _.has(record, 'mlcategory')) { - const jobId = record.job_id; - - // mlcategory in the source record will be an array - // - use first value (will only ever be more than one if influenced by category other than by/partition/over). - const categoryId = record.mlcategory[0]; - ml.results.getCategoryDefinition(jobId, categoryId) - .then((resp) => { - // Prefix each of the terms with '+' so that the Elasticsearch Query String query - // run in a drilldown Kibana dashboard has to match on all terms. - const termsArray = _.map(resp.terms.split(' '), (term) => { return '+' + term; }); - record.mlcategoryterms = termsArray.join(' '); - record.mlcategoryregex = resp.regex; - - // Replace any tokens in the configured url_value with values from the source record, - // and then open link in a new tab/window. - const urlPath = replaceStringTokens(customUrl.url_value, record, true); - $window.open(urlPath, '_blank'); - - }).catch((resp) => { - console.log('openCustomUrl(): error loading categoryDefinition:', resp); - }); - - } else { - // Replace any tokens in the configured url_value with values from the source record, - // and then open link in a new tab/window. - const urlPath = getUrlForRecord(customUrl, record); - $window.open(urlPath, '_blank'); - } - - }; - - scope.filter = function (field, value, operator) { - mlAnomaliesTableService.filterChange.changed(field, value, operator); - }; - - function updateTableData() { - let summaryRecords = []; - if (scope.isShowingAggregatedData()) { - // Aggregate the anomaly data by time and detector, and entity (by/over). - summaryRecords = aggregateAnomalies(); - } else { - // Show all anomaly records. - const interval = mlSelectIntervalService.state.get('interval'); - scope.momentInterval = interval.val; - const threshold = mlSelectSeverityService.state.get('threshold'); - const filteredRecords = _.filter(scope.anomalyRecords, (record) => { - return Number(record.record_score) >= threshold.val; - }); - - _.each(filteredRecords, (record) => { - const detectorIndex = record.detector_index; - const jobId = record.job_id; - let detector = record.function_description; - if ((_.has(mlJobService.detectorsByJob, jobId)) && (detectorIndex < mlJobService.detectorsByJob[jobId].length)) { - detector = mlJobService.detectorsByJob[jobId][detectorIndex].detector_description; - } - - const displayRecord = { - 'time': record[scope.timeFieldName], - 'max severity': record.record_score, - 'detector': detector, - 'jobId': jobId, - 'source': record - }; - - const entityName = getEntityFieldName(record); - if (entityName !== undefined) { - displayRecord.entityName = entityName; - displayRecord.entityValue = getEntityFieldValue(record); - } - - if (_.has(record, 'partition_field_name')) { - displayRecord.partitionFieldName = record.partition_field_name; - displayRecord.partitionFieldValue = record.partition_field_value; - } - - if (_.has(record, 'influencers')) { - const influencers = []; - const sourceInfluencers = _.sortBy(record.influencers, 'influencer_field_name'); - _.each(sourceInfluencers, (influencer) => { - const influencerFieldName = influencer.influencer_field_name; - _.each(influencer.influencer_field_values, (influencerFieldValue) => { - const influencerToAdd = {}; - influencerToAdd[influencerFieldName] = influencerFieldValue; - influencers.push(influencerToAdd); - }); - }); - displayRecord.influencers = influencers; - } - - const functionDescription = _.get(record, 'function_description', ''); - if (showActualForFunction(functionDescription) === true) { - if (_.has(record, 'actual')) { - displayRecord.actual = record.actual; - } else { - // If only a single cause, copy values to the top level. - if (_.get(record, 'causes', []).length === 1) { - const cause = _.first(record.causes); - displayRecord.actual = cause.actual; - } - } - } - if (showTypicalForFunction(functionDescription) === true) { - if (_.has(record, 'typical')) { - displayRecord.typical = record.typical; - } else { - // If only a single cause, copy values to the top level. - if (_.get(record, 'causes', []).length === 1) { - const cause = _.first(record.causes); - displayRecord.typical = cause.typical; - } - } - } - - if (_.has(mlJobService.customUrlsByJob, jobId)) { - displayRecord.customUrls = mlJobService.customUrlsByJob[jobId]; - } - - summaryRecords.push(displayRecord); - - }); - } - - _.invoke(scope.rowScopes, '$destroy'); - scope.rowScopes.length = 0; - - const showExamples = _.some(summaryRecords, { 'entityName': 'mlcategory' }); - if (showExamples) { - // Obtain the list of categoryIds by jobId for which we need to obtain the examples. - // Note category examples will not be displayed if mlcategory is used just an - // influencer or as a partition field in a config with other by/over fields. - const categoryRecords = _.where(summaryRecords, { entityName: 'mlcategory' }); - const categoryIdsByJobId = {}; - _.each(categoryRecords, (record) => { - if (!_.has(categoryIdsByJobId, record.jobId)) { - categoryIdsByJobId[record.jobId] = []; - } - categoryIdsByJobId[record.jobId].push(record.entityValue); - }); - loadCategoryExamples(categoryIdsByJobId); - } else { - scope.categoryExamplesByJob = {}; - } - - // Only show columns in the table which exist in the results. - scope.table.columns = getPaginatedTableColumns(summaryRecords); - - // Sort by severity by default. - summaryRecords = (_.sortBy(summaryRecords, 'max severity')).reverse(); - scope.table.rows = summaryRecords.map((record) => { - return createTableRow(record); - }); - - } - - function aggregateAnomalies() { - // Aggregate the anomaly data by time, detector, and entity (by/over/partition). - // TODO - do we want to aggregate by job too, in cases where different jobs - // have detectors with the same description. - console.log('aggregateAnomalies(): number of anomalies to aggregate:', scope.anomalyRecords.length); - - if (scope.anomalyRecords.length === 0) { - return []; - } - - // Determine the aggregation interval - records in scope are in descending time order. - const interval = mlSelectIntervalService.state.get('interval'); - if (interval.val === 'auto') { - const earliest = moment(_.last(scope.anomalyRecords)[scope.timeFieldName]); - const latest = moment(_.first(scope.anomalyRecords)[scope.timeFieldName]); - const daysDiff = latest.diff(earliest, 'days'); - scope.momentInterval = (daysDiff < 2 ? 'hour' : 'day'); - } else { - scope.momentInterval = interval.val; - } - - // Only show records passing the severity threshold. - const threshold = mlSelectSeverityService.state.get('threshold'); - const filteredRecords = _.filter(scope.anomalyRecords, (record) => { - - return Number(record.record_score) >= threshold.val; - }); - - const aggregatedData = {}; - _.each(filteredRecords, (record) => { - - // Use moment.js to get start of interval. This will use browser timezone. - // TODO - support choice of browser or UTC timezone once functionality is in Kibana. - const roundedTime = moment(record[scope.timeFieldName]).startOf(scope.momentInterval).valueOf(); - if (!_.has(aggregatedData, roundedTime)) { - aggregatedData[roundedTime] = {}; - } - - // Aggregate by detector - default to functionDescription if no description available. - const detectorIndex = record.detector_index; - const jobId = record.job_id; - let detector = record.function_description; - if ((_.has(mlJobService.detectorsByJob, jobId)) && (detectorIndex < mlJobService.detectorsByJob[jobId].length)) { - detector = mlJobService.detectorsByJob[jobId][detectorIndex].detector_description; - } - const detectorsAtTime = aggregatedData[roundedTime]; - if (!_.has(detectorsAtTime, detector)) { - detectorsAtTime[detector] = {}; - } - - // Now add an object for the anomaly with the highest anomaly score per entity. - // For the choice of entity, look in order for byField, overField, partitionField. - // If no by/over/partition, default to an empty String. - const entitiesForDetector = detectorsAtTime[detector]; - - // TODO - are we worried about different byFields having the same - // value e.g. host=server1 and machine=server1? - let entity = getEntityFieldValue(record); - if (entity === undefined) { - entity = ''; - } - if (!_.has(entitiesForDetector, entity)) { - entitiesForDetector[entity] = record; - } else { - const score = record.record_score; - if (score > entitiesForDetector[entity].record_score) { - entitiesForDetector[entity] = record; - } - } - }); - - console.log('aggregateAnomalies() aggregatedData is:', aggregatedData); - - // Flatten the aggregatedData to give a list of records with the highest score per bucketed time / detector. - const summaryRecords = []; - _.each(aggregatedData, (timeDetectors, roundedTime) => { - _.each(timeDetectors, (entityDetectors, detector) => { - _.each(entityDetectors, (record, entity) => { - const displayRecord = { - 'time': +roundedTime, - 'max severity': record.record_score, - 'detector': detector, - 'jobId': record.job_id, - 'source': record - }; - - const entityName = getEntityFieldName(record); - if (entityName !== undefined) { - displayRecord.entityName = entityName; - displayRecord.entityValue = entity; - } - - if (_.has(record, 'partition_field_name')) { - displayRecord.partitionFieldName = record.partition_field_name; - displayRecord.partitionFieldValue = record.partition_field_value; - } - - if (_.has(record, 'influencers')) { - const influencers = []; - const sourceInfluencers = _.sortBy(record.influencers, 'influencer_field_name'); - _.each(sourceInfluencers, (influencer) => { - const influencerFieldName = influencer.influencer_field_name; - _.each(influencer.influencer_field_values, (influencerFieldValue) => { - const influencerToAdd = {}; - influencerToAdd[influencerFieldName] = influencerFieldValue; - influencers.push(influencerToAdd); - }); - }); - displayRecord.influencers = influencers; - } - - // Copy actual and typical values to the top level for display. - const functionDescription = _.get(record, 'function_description', ''); - if (showActualForFunction(functionDescription) === true) { - if (_.has(record, 'actual')) { - displayRecord.actual = record.actual; - } else { - // If only a single cause, copy value to the top level. - if (_.get(record, 'causes', []).length === 1) { - const cause = _.first(record.causes); - displayRecord.actual = cause.actual; - } - } - } - if (showTypicalForFunction(functionDescription) === true) { - if (_.has(record, 'typical')) { - displayRecord.typical = record.typical; - } else { - // If only a single cause, copy value to the top level. - if (_.get(record, 'causes', []).length === 1) { - const cause = _.first(record.causes); - displayRecord.typical = cause.typical; - } - } - } - - if (_.has(mlJobService.customUrlsByJob, record.job_id)) { - displayRecord.customUrls = mlJobService.customUrlsByJob[record.job_id]; - } - - summaryRecords.push(displayRecord); - - }); - }); - }); - - return summaryRecords; - - } - - function getPaginatedTableColumns(summaryRecords) { - // Builds the list of columns for use in the paginated table: - // row expand arrow - // time - // max severity - // detector - // found for (if by/over/partition) - // influenced by (if influencers) - // actual - // typical - // description (how actual compares to typical) - // job_id - // links (if custom URLs configured or drilldown functionality) - // category examples (if by mlcategory) - const paginatedTableColumns = [ - { title: '', sortable: false, class: 'col-expand-arrow' }, - { title: 'time', sortable: true }]; - - if (scope.isShowingAggregatedData()) { - paginatedTableColumns.push({ title: 'max severity', sortable: true }); - } else { - paginatedTableColumns.push({ title: 'severity', sortable: true }); - } - - paginatedTableColumns.push({ title: 'detector', sortable: true }); - - const showEntity = _.some(summaryRecords, 'entityValue'); - const showInfluencers = _.some(summaryRecords, 'influencers'); - const showActual = _.some(summaryRecords, 'actual'); - const showTypical = _.some(summaryRecords, 'typical'); - const showExamples = _.some(summaryRecords, { 'entityName': 'mlcategory' }); - const showLinks = ((scope.showViewSeriesLink === true) && - _.some(summaryRecords, (record) => { - const job = mlJobService.getJob(record.jobId); - return isTimeSeriesViewDetector(job, record.source.detector_index); - })) || showExamples === true || _.some(summaryRecords, 'customUrls'); - - if (showEntity === true) { - paginatedTableColumns.push({ title: 'found for', sortable: true }); - } - if (showInfluencers === true) { - paginatedTableColumns.push({ title: 'influenced by', sortable: true }); - } - if (showActual === true) { - paginatedTableColumns.push({ title: 'actual', sortable: true }); - } - if (showTypical === true) { - paginatedTableColumns.push({ title: 'typical', sortable: true }); - - // Assume that if we are showing typical, there will be an actual too, - // so we can add a column to describe how actual compares to typical. - const nonTimeOfDayOrWeek = _.some(summaryRecords, (record) => { - const summaryRecFunc = record.source.function; - return summaryRecFunc !== 'time_of_day' && summaryRecFunc !== 'time_of_week'; - }); - if (nonTimeOfDayOrWeek === true) { - paginatedTableColumns.push({ title: 'description', sortable: true }); - } - } - paginatedTableColumns.push({ title: 'job ID', sortable: true }); - if (showLinks === true) { - paginatedTableColumns.push({ title: 'links', sortable: false }); - } - if (showExamples === true) { - paginatedTableColumns.push({ title: 'category examples', sortable: false }); - } - - return paginatedTableColumns; - } - - - function createTableRow(record) { - const rowScope = scope.$new(); - rowScope.expandable = true; - rowScope.expandElement = 'ml-anomalies-table-expanded-row'; - rowScope.record = record; - rowScope.isShowingAggregatedData = scope.isShowingAggregatedData(); - - rowScope.initRow = function () { - if (_.has(record, 'entityValue') && record.entityName === 'mlcategory') { - // Obtain the category definition and display the examples in the expanded row. - ml.results.getCategoryDefinition(record.jobId, record.entityValue) - .then((resp) => { - rowScope.categoryDefinition = { - 'examples': _.slice(resp.examples, 0, Math.min(resp.examples.length, MAX_NUMBER_CATEGORY_EXAMPLES)) }; - }).catch((resp) => { - console.log('Anomalies table createTableRow(): error loading categoryDefinition:', resp); - }); - } - - rowScope.$broadcast('initRow', record); - }; - - rowScope.mouseenterRow = function () { - // Publish that a record is being hovered over, so that the corresponding marker - // in the model plot chart can be highlighted. - mlAnomaliesTableService.anomalyRecordMouseenter.changed(record); - }; - - rowScope.mouseleaveRow = function () { - // Publish that a record is no longer being hovered over, so that the corresponding marker in the - // model plot chart can be unhighlighted. - mlAnomaliesTableService.anomalyRecordMouseleave.changed(record); - }; - - // Create a table row with the following columns: - // row expand arrow - // time - // max severity - // detector - // found for (if by/over/partition) - // influenced by (if influencers) - // actual - // typical - // description (how actual compares to typical) - // job_id - // links (if customUrls configured or drilldown to Single Metric) - // category examples (if by mlcategory) - const addEntity = _.findWhere(scope.table.columns, { 'title': 'found for' }); - const addInfluencers = _.findWhere(scope.table.columns, { 'title': 'influenced by' }); - - const addActual = _.findWhere(scope.table.columns, { 'title': 'actual' }); - const addTypical = _.findWhere(scope.table.columns, { 'title': 'typical' }); - const addDescription = _.findWhere(scope.table.columns, { 'title': 'description' }); - const addExamples = _.findWhere(scope.table.columns, { 'title': 'category examples' }); - const addLinks = _.findWhere(scope.table.columns, { 'title': 'links' }); - const fieldFormat = mlFieldFormatService.getFieldFormat(record.jobId, record.source.detector_index); - - const tableRow = [ - { - markup: openRowArrow, - scope: rowScope - }, - { - markup: formatTimestamp(record.time), - value: record.time - }, - { - markup: parseInt(record['max severity']) >= 1 ? - ' ' + Math.floor(record['max severity']) : - ' < 1', - value: record['max severity'] - }, - { - markup: mlEscape(record.detector), - value: record.detector - } - ]; - - if (addEntity !== undefined) { - if (_.has(record, 'entityValue')) { - if (record.entityName !== 'mlcategory') { - // Escape single quotes and backslash characters in the HTML for the event handlers. - const safeEntityName = mlEscape(record.entityName.replace(/(['\\])/g, '\\$1')); - const safeEntityValue = mlEscape(record.entityValue.replace(/(['\\])/g, '\\$1')); - - tableRow.push({ - markup: mlEscape(record.entityValue) + - ' ' + - ' ', - value: record.entityValue, - scope: rowScope - }); - } else { - tableRow.push({ - markup: 'mlcategory ' + record.entityValue, - value: record.entityValue - }); - } - } else { - tableRow.push({ - markup: '', - value: '' - }); - } - } - - if (addInfluencers !== undefined) { - if (_.has(record, 'influencers')) { - const cellMarkup = ``; - tableRow.push({ - markup: cellMarkup, - value: cellMarkup, - scope: rowScope - }); - } else { - tableRow.push({ - markup: '', - value: '' - }); - } - } - - if (addActual !== undefined) { - if (_.has(record, 'actual')) { - tableRow.push({ - markup: formatValue(record.actual, record.source.function, fieldFormat), - // Store the unformatted value as a number so that sorting works correctly. - // actual and typical values in anomaly record results will be arrays. - value: Array.isArray(record.actual) && record.actual.length === 1 ? - Number(record.actual[0]) : String(record.actual), - scope: rowScope }); - } else { - tableRow.push({ markup: '', value: '' }); - } - } - if (addTypical !== undefined) { - if (_.has(record, 'typical')) { - const typicalVal = Array.isArray(record.typical) && record.typical.length === 1 ? - Number(record.typical[0]) : String(record.typical); - tableRow.push({ - markup: formatValue(record.typical, record.source.function, fieldFormat), - value: typicalVal, - scope: rowScope }); - - if (addDescription !== undefined) { - // Assume there is an actual value if there is a typical, - // and add a description cell if not time_of_week/day. - const detectorFunc = record.source.function; - if (detectorFunc !== 'time_of_week' && detectorFunc !== 'time_of_day') { - let factor = 0; - if (Array.isArray(record.typical) && record.typical.length === 1 && - Array.isArray(record.actual) && record.actual.length === 1) { - const actualVal = Number(record.actual[0]); - factor = (actualVal > typicalVal) ? actualVal / typicalVal : typicalVal / actualVal; - } - - tableRow.push({ - markup: ``, - value: Math.abs(factor), - scope: rowScope }); - } else { - tableRow.push({ markup: '', value: '' }); - } - } - } else { - tableRow.push({ markup: '', value: '' }); - if (addDescription !== undefined) { - tableRow.push({ markup: '', value: '' }); - } - } - } - - tableRow.push({ markup: record.jobId, value: record.jobId }); - - if (addLinks !== undefined) { - const job = mlJobService.getJob(record.jobId); - rowScope.showViewSeriesLink = scope.showViewSeriesLink === true && - isTimeSeriesViewDetector(job, record.source.detector_index); - rowScope.showViewExamplesLink = (_.get(record, 'entityName') === 'mlcategory'); - if (_.has(record, 'customUrls') || rowScope.showViewSeriesLink === true - || rowScope.showViewExamplesLink) { - rowScope.customUrls = record.customUrls; - rowScope.source = record.source; - - tableRow.push({ - markup: linkControlsHtml, - scope: rowScope - }); - } else { - tableRow.push({ - markup: '', - value: '' - }); - } - } - - if (addExamples !== undefined) { - if (record.entityName === 'mlcategory') { - tableRow.push({ markup: '{{item}}', scope: rowScope }); - } else { - tableRow.push({ markup: '', value: '' }); - } - } - - scope.rowScopes.push(rowScope); - - return tableRow; - - } - - function loadCategoryExamples(categoryIdsByJobId) { - // Load the example events for the specified map of job_ids and categoryIds from Elasticsearch. - scope.categoryExamplesByJob = {}; - _.each(categoryIdsByJobId, (categoryIds, jobId) => { - ml.results.getCategoryExamples(jobId, categoryIds, MAX_NUMBER_CATEGORY_EXAMPLES) - .then((resp) => { - scope.categoryExamplesByJob[jobId] = resp; - }).catch((resp) => { - console.log('Anomalies table - error getting category examples:', resp); - }); - }); - } - - function formatTimestamp(epochMs) { - const time = moment(epochMs); - if (scope.momentInterval === 'hour') { - return time.format('MMMM Do YYYY, HH:mm'); - } else if (scope.momentInterval === 'second') { - return time.format('MMMM Do YYYY, HH:mm:ss'); - } else { - return time.format('MMMM Do YYYY'); - } - } - +const module = uiModules.get('apps/ml', ['react']); + +import { AnomaliesTable } from './anomalies_table'; + +module.directive('mlAnomaliesTable', function ($injector) { + const timefilter = $injector.get('timefilter'); + const reactDirective = $injector.get('reactDirective'); + + return reactDirective( + AnomaliesTable, + [ + ['filter', { watchDepth: 'reference' }], + ['tableData', { watchDepth: 'reference' }] + ], + { restrict: 'E' }, + { + timefilter } - }; + ); }); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_links.html b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_links.html deleted file mode 100644 index f6576d233d73..000000000000 --- a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_links.html +++ /dev/null @@ -1,20 +0,0 @@ - - diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_service.js b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_service.js index bfdecbad9688..2c7147650c22 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_service.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/anomalies_table_service.js @@ -11,16 +11,14 @@ * anomalies table component. */ -import { uiModules } from 'ui/modules'; -const module = uiModules.get('apps/ml'); - import { listenerFactoryProvider } from 'plugins/ml/factories/listener_factory'; -module.service('mlAnomaliesTableService', function () { +class AnomaliesTableService { + constructor() { + const listenerFactory = listenerFactoryProvider(); + this.anomalyRecordMouseenter = listenerFactory(); + this.anomalyRecordMouseleave = listenerFactory(); + } +} - const listenerFactory = listenerFactoryProvider(); - this.anomalyRecordMouseenter = listenerFactory(); - this.anomalyRecordMouseleave = listenerFactory(); - this.filterChange = listenerFactory(); - -}); +export const mlAnomaliesTableService = new AnomaliesTableService(); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/anomaly_details.js b/x-pack/plugins/ml/public/components/anomalies_table/anomaly_details.js new file mode 100644 index 000000000000..fdd12e1e5960 --- /dev/null +++ b/x-pack/plugins/ml/public/components/anomalies_table/anomaly_details.js @@ -0,0 +1,326 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +/* + * React component for displaying details of an anomaly in the expanded row section + * of the anomalies table. + */ + +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; +import _ from 'lodash'; + +import { + EuiDescriptionList, + EuiIcon, + EuiLink, + EuiSpacer, + EuiText +} from '@elastic/eui'; +import { formatDate } from '@elastic/eui/lib/services/format'; + +import { EntityCell } from './entity_cell'; +import { + getSeverity, + showActualForFunction, + showTypicalForFunction +} from 'plugins/ml/../common/util/anomaly_utils'; +import { formatValue } from 'plugins/ml/formatters/format_value'; + +const TIME_FIELD_NAME = 'timestamp'; + + +function getFilterEntity(entityName, entityValue, filter) { + return ( + + ); +} + +function getDetailsItems(anomaly, examples, filter) { + const source = anomaly.source; + + // TODO - when multivariate analyses are more common, + // look in each cause for a 'correlatedByFieldValue' field, + let causes = []; + const sourceCauses = source.causes || []; + let singleCauseByFieldName = undefined; + let singleCauseByFieldValue = undefined; + if (sourceCauses.length === 1) { + // Metrics and probability will already have been placed at the top level. + // If cause has byFieldValue, move it to a top level fields for display. + if (sourceCauses[0].by_field_name !== undefined) { + singleCauseByFieldName = sourceCauses[0].by_field_name; + singleCauseByFieldValue = sourceCauses[0].by_field_value; + } + } else { + causes = sourceCauses.map((cause) => { + const simplified = _.pick(cause, 'typical', 'actual', 'probability'); + // Get the 'entity field name/value' to display in the cause - + // For by and over, use by_field_name/value (over_field_name/value are in the top level fields) + // For just an 'over' field - the over_field_name/value appear in both top level and cause. + simplified.entityName = _.has(cause, 'by_field_name') ? cause.by_field_name : cause.over_field_name; + simplified.entityValue = _.has(cause, 'by_field_value') ? cause.by_field_value : cause.over_field_value; + return simplified; + }); + } + + const items = []; + if (source.partition_field_value !== undefined) { + items.push({ + title: source.partition_field_name, + description: getFilterEntity(source.partition_field_name, source.partition_field_value, filter) + }); + } + + if (source.by_field_value !== undefined) { + items.push({ + title: source.by_field_name, + description: getFilterEntity(source.by_field_name, source.by_field_value, filter) + }); + } + + if (singleCauseByFieldName !== undefined) { + // Display byField of single cause. + items.push({ + title: singleCauseByFieldName, + description: getFilterEntity(singleCauseByFieldName, singleCauseByFieldValue, filter) + }); + } + + if (source.over_field_value !== undefined) { + items.push({ + title: source.over_field_name, + description: getFilterEntity(source.over_field_name, source.over_field_value, filter) + }); + } + + const anomalyTime = source[TIME_FIELD_NAME]; + let timeDesc = `${formatDate(anomalyTime, 'MMMM Do YYYY, HH:mm:ss')}`; + if (source.bucket_span !== undefined) { + const anomalyEndTime = anomalyTime + (source.bucket_span * 1000); + timeDesc += ` to ${formatDate(anomalyEndTime, 'MMMM Do YYYY, HH:mm:ss')}`; + } + items.push({ + title: 'time', + description: timeDesc + }); + + if (examples !== undefined && examples.length > 0) { + examples.forEach((example, index) => { + const title = (index === 0) ? 'category examples' : ''; + items.push({ title, description: example }); + }); + } + + items.push({ + title: 'function', + description: (source.function !== 'metric') ? source.function : source.function_description + }); + + if (source.field_name !== undefined) { + items.push({ + title: 'fieldName', + description: source.field_name + }); + } + + const functionDescription = source.function_description || ''; + if (anomaly.actual !== undefined && showActualForFunction(functionDescription) === true) { + items.push({ + title: 'actual', + description: formatValue(anomaly.actual, source.function) + }); + } + + if (anomaly.typical !== undefined && showTypicalForFunction(functionDescription) === true) { + items.push({ + title: 'typical', + description: formatValue(anomaly.typical, source.function) + }); + } + + items.push({ + title: 'job ID', + description: anomaly.jobId + }); + + items.push({ + title: 'probability', + description: source.probability + }); + + // If there was only one cause, the actual, typical and by_field + // will already have been added for display. + if (causes.length > 1) { + causes.forEach((cause, index) => { + const title = (index === 0) ? `${cause.entityName} values` : ''; + let description = `${cause.entityValue} (actual ${formatValue(cause.actual, source.function)}, `; + description += `typical ${formatValue(cause.typical, source.function)}, probability ${cause.probability})`; + items.push({ title, description }); + }); + } + + return items; +} + +export class AnomalyDetails extends Component { + + constructor(props) { + super(props); + this.state = { + showAllInfluencers: false + }; + } + + toggleAllInfluencers() { + this.setState({ showAllInfluencers: !this.state.showAllInfluencers }); + } + + renderDescription() { + const anomaly = this.props.anomaly; + const source = anomaly.source; + + let anomalyDescription = `${getSeverity(anomaly.severity)} anomaly in ${anomaly.detector}`; + if (anomaly.entityName !== undefined) { + anomalyDescription += ` found for ${anomaly.entityName} ${anomaly.entityValue}`; + } + + if ((source.partition_field_name !== undefined) && + (source.partition_field_name !== anomaly.entityName)) { + anomalyDescription += ` detected in ${source.partition_field_name}`; + anomalyDescription += ` ${source.partition_field_value}`; + } + + // Check for a correlatedByFieldValue in the source which will be present for multivariate analyses + // where the record is anomalous due to relationship with another 'by' field value. + let mvDescription = undefined; + if (source.correlated_by_field_value !== undefined) { + mvDescription = `multivariate correlations found in ${source.by_field_name}; `; + mvDescription += `${source.by_field_value} is considered anomalous given ${source.correlated_by_field_value}`; + } + return ( + + +
Description
+ {anomalyDescription} +
+ {(mvDescription !== undefined) && + + {mvDescription} + + } +
+ ); + } + + renderDetails() { + const detailItems = getDetailsItems(this.props.anomaly, this.props.examples, this.props.filter); + const isInterimResult = _.get(this.props.anomaly, 'source.is_interim', false); + return ( + + + {this.props.isAggregatedData === true ? ( +
Details on highest severity anomaly
+ ) : ( +
Anomaly details
+ )} + {isInterimResult === true && + + Interim result + + } +
+ +
+ ); + } + + renderInfluencers() { + const anomalyInfluencers = this.props.anomaly.influencers; + const listItems = []; + let othersCount = 0; + let numToDisplay = 0; + if (anomalyInfluencers !== undefined) { + numToDisplay = (this.state.showAllInfluencers === true) ? + anomalyInfluencers.length : Math.min(this.props.influencersLimit, anomalyInfluencers.length); + othersCount = Math.max(anomalyInfluencers.length - numToDisplay, 0); + + if (othersCount === 1) { + // Display the 1 extra influencer as displaying "and 1 more" would also take up a line. + numToDisplay++; + othersCount = 0; + } + + for (let i = 0; i < numToDisplay; i++) { + Object.keys(anomalyInfluencers[i]).forEach((influencerFieldName) => { + listItems.push({ + title: influencerFieldName, + description: anomalyInfluencers[i][influencerFieldName] + }); + }); + } + } + + if (listItems.length > 0) { + return ( + + + +
Influencers
+
+ + {othersCount > 0 && + this.toggleAllInfluencers()} + > + and {othersCount} more + + } + {numToDisplay > (this.props.influencersLimit + 1) && + this.toggleAllInfluencers()} + > + show less + + } +
+ ); + } + } + + render() { + + return ( +
+ {this.renderDescription()} + + {this.renderDetails()} + {this.renderInfluencers()} +
+ ); + } +} + +AnomalyDetails.propTypes = { + anomaly: PropTypes.object.isRequired, + examples: PropTypes.array, + isAggregatedData: PropTypes.bool, + filter: PropTypes.func, + influencersLimit: PropTypes.number +}; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/description_cell.js b/x-pack/plugins/ml/public/components/anomalies_table/description_cell.js new file mode 100644 index 000000000000..034bb905aef4 --- /dev/null +++ b/x-pack/plugins/ml/public/components/anomalies_table/description_cell.js @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +import PropTypes from 'prop-types'; +import React from 'react'; + +import { + EuiFlexGroup, + EuiFlexItem, + EuiIcon, + EuiText +} from '@elastic/eui'; + +import { getMetricChangeDescription } from 'plugins/ml/formatters/metric_change_description'; + +/* + * Component for rendering the description cell in the anomalies table, which provides a + * concise description of how the actual value of an anomaly compares to the typical value. + */ +export function DescriptionCell({ actual, typical }) { + + const { + iconType, + message + } = getMetricChangeDescription(actual, typical); + + return ( + + {iconType !== undefined && + + + + } + + +

{message}

+
+
+
+ ); +} + +DescriptionCell.propTypes = { + actual: PropTypes.oneOfType([PropTypes.array, PropTypes.number]), + typical: PropTypes.oneOfType([PropTypes.array, PropTypes.number]) +}; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/entity_cell.js b/x-pack/plugins/ml/public/components/anomalies_table/entity_cell.js new file mode 100644 index 000000000000..ca6d3cf7e68c --- /dev/null +++ b/x-pack/plugins/ml/public/components/anomalies_table/entity_cell.js @@ -0,0 +1,56 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +import PropTypes from 'prop-types'; +import React from 'react'; + +import { + EuiButtonIcon, + EuiToolTip +} from '@elastic/eui'; + +/* + * Component for rendering an entity cell in the anomalies table, displaying the value + * of the 'partition', 'by' or 'over' field, and optionally links for adding or removing + * a filter on this entity. + */ +export function EntityCell({ entityName, entityValue, filter }) { + const valueText = (entityName !== 'mlcategory') ? entityValue : `mlcategory ${entityValue}`; + return ( + + {valueText} + {filter !== undefined && entityName !== undefined && entityValue !== undefined && + + + filter(entityName, entityValue, '+')} + iconType="plusInCircle" + aria-label="Add filter" + /> + + + filter(entityName, entityValue, '-')} + iconType="minusInCircle" + aria-label="Remove filter" + /> + + + } + + ); +} + +EntityCell.propTypes = { + entityName: PropTypes.string, + entityValue: PropTypes.any, + filter: PropTypes.func +}; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row.html b/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row.html deleted file mode 100644 index df613ab198fa..000000000000 --- a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row.html +++ /dev/null @@ -1,108 +0,0 @@ -
- Description: - {{description}} - {{multiVariateDescription}} - - Details on highest severity anomaly: - Anomaly Details: -
Interim result
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
{{record.source.partition_field_name}}:{{record.source.partition_field_value}} - - -
{{record.source.by_field_name}}:{{record.source.by_field_value}} - - -
{{singleCauseByFieldName}}:{{singleCauseByFieldValue}} - - -
{{record.source.over_field_name}}:{{record.source.over_field_value}} - - -
time:{{anomalyTime}} to {{anomalyEndTime}}
time:{{anomalyTime}}
category examples: {{example}}
function:{{record.source.function}}{{record.source.function_description}}
fieldName:{{record.source.field_name}}
actual:{{actual | formatValue:record.source.function}}
typical:{{typical | formatValue:record.source.function}}
job ID:{{record.jobId}} -
probability:{{record.source.probability}}
{{cause.entityName}} values: {{cause.entityValue}} (actual {{cause.actual | formatValue:record.source.function}}, typical {{cause.typical | formatValue:record.source.function}}, probability {{cause.probability}})
- - Influenced by: - - - - - - - - - - - -
{{influencer.name}}{{influencer.value}}
- -
- -
- -
diff --git a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js b/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js deleted file mode 100644 index bd3422f26b1a..000000000000 --- a/x-pack/plugins/ml/public/components/anomalies_table/expanded_row/expanded_row_directive.js +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - - -/* - * Angular directive for rendering the expanded row content in the - * Machine Learning anomalies table. It displays more details on the - * anomaly summarized in the row, including field names, - * actual and typical values for the analyzed metric, - * plus causes and examples events according to the detector configuration. - */ - -import _ from 'lodash'; -import moment from 'moment'; - -import template from './expanded_row.html'; -import { - getSeverity, - showActualForFunction, - showTypicalForFunction -} from 'plugins/ml/../common/util/anomaly_utils'; -import 'plugins/ml/formatters/format_value'; - -import { uiModules } from 'ui/modules'; -const module = uiModules.get('apps/ml'); - -module.directive('mlAnomaliesTableExpandedRow', function () { - - function link(scope) { - scope.record = scope.$parent.record; - scope.filter = scope.$parent.filter; - scope.filteringEnabled = scope.$parent.filteringEnabled; - scope.isShowingAggregatedData = scope.$parent.isShowingAggregatedData; - scope.influencersLimit = scope.$parent.influencersLimit; - scope.influencersNumToDislay = scope.influencersLimit; - - const timeFieldName = 'timestamp'; - const momentTime = moment(scope.record.source[timeFieldName]); - scope.anomalyTime = momentTime.format('MMMM Do YYYY, HH:mm:ss'); - if (_.has(scope.record.source, 'bucket_span')) { - scope.anomalyEndTime = momentTime.add(scope.record.source.bucket_span, 's').format('MMMM Do YYYY, HH:mm:ss'); - } - - scope.$on('initRow', () => { - // Only build the description and details on metric values, - // causes and influencers when the row is first expanded. - buildContent(); - }); - - scope.toggleAllInfluencers = function () { - if (_.has(scope.record, 'influencers')) { - const recordInfluencers = scope.record.influencers; - if (scope.influencers.length === recordInfluencers.length) { - scope.influencersNumToDislay = scope.influencersLimit; - } else { - scope.influencersNumToDislay = recordInfluencers.length; - } - buildInfluencers(); - } - }; - - if (scope.$parent.open === true) { - // Build the content if the row was already open before re-render (e.g. when sorting), - buildContent(); - } - - if (_.has(scope.record, 'entityValue') && scope.record.entityName === 'mlcategory') { - // For categorization results, controller will obtain the definition when the - // row is first expanded and place the categoryDefinition in the row scope. - const unbindWatch = scope.$parent.$watch('categoryDefinition', (categoryDefinition) => { - if (categoryDefinition !== undefined) { - scope.examples = categoryDefinition.examples; - unbindWatch(); - } - }); - } - - function buildContent() { - buildDescription(); - buildMetrics(); - buildCauses(); - buildInfluencers(); - } - - function buildDescription() { - const record = scope.record; - let rowDescription = getSeverity(record.source.record_score) + ' anomaly in ' + record.detector; - - if (_.has(record, 'entityName')) { - rowDescription += ' found for ' + record.entityName; - rowDescription += ' '; - rowDescription += record.entityValue; - } - - if (_.has(record.source, 'partition_field_name') && (record.source.partition_field_name !== record.entityName)) { - rowDescription += ' detected in ' + record.source.partition_field_name; - rowDescription += ' '; - rowDescription += record.source.partition_field_value; - } - - scope.description = rowDescription; - - // Check for a correlatedByFieldValue in the source which will be present for multivariate analyses - // where the record is anomalous due to relationship with another 'by' field value. - if (_.has(record.source, 'correlated_by_field_value')) { - let mvDescription = 'multivariate correlations found in '; - mvDescription += record.source.by_field_name; - mvDescription += '; '; - mvDescription += record.source.by_field_value; - mvDescription += ' is considered anomalous given '; - mvDescription += record.source.correlated_by_field_value; - scope.multiVariateDescription = mvDescription; - } - - - // Display a warning below the description if the record is an interim result. - scope.isInterim = _.get(record, 'source.is_interim', false); - } - - function buildMetrics() { - const record = scope.record; - const functionDescription = _.get(record, 'source.function_description', ''); - if (showActualForFunction(functionDescription) === true) { - if (!_.has(scope.record.source, 'causes')) { - scope.actual = record.source.actual; - } else { - const causes = scope.record.source.causes; - if (causes.length === 1) { - // If only one 'cause', move value to top level. - const cause = _.first(causes); - scope.actual = cause.actual; - } - } - } - if (showTypicalForFunction(functionDescription) === true) { - if (!_.has(scope.record.source, 'causes')) { - scope.typical = record.source.typical; - } else { - const causes = scope.record.source.causes; - if (causes.length === 1) { - // If only one 'cause', move value to top level. - const cause = _.first(causes); - scope.typical = cause.typical; - } - } - } - } - - function buildCauses() { - if (_.has(scope.record.source, 'causes')) { - const causes = scope.record.source.causes; - - // TODO - build different information depending on whether function is rare, freq_rare or another. - - // TODO - look in each cause for a 'correlatedByFieldValue' field, - // and if so, add to causes scope object for rendering in the template. - if (causes.length === 1) { - // Metrics and probability will already have been placed at the top level. - // If cause has byFieldValue, move it to a top level fields for display. - const cause = _.first(causes); - if (_.has(cause, 'by_field_name')) { - scope.singleCauseByFieldName = cause.by_field_name; - scope.singleCauseByFieldValue = cause.by_field_value; - } - } else { - scope.causes = _.map(causes, (cause) => { - const simplified = _.pick(cause, 'typical', 'actual', 'probability'); - // Get the 'entity field name/value' to display in the cause - - // For by and over, use by_field_name/Value (over_field_name/Value are in the toplevel fields) - // For just an 'over' field - the over_field_name/Value appear in both top level and cause. - simplified.entityName = _.has(cause, 'by_field_name') ? cause.by_field_name : cause.over_field_name; - simplified.entityValue = _.has(cause, 'by_field_value') ? cause.by_field_value : cause.over_field_value; - return simplified; - }); - } - - } - } - - function buildInfluencers() { - if (_.has(scope.record, 'influencers')) { - const recordInfluencers = scope.record.influencers; - scope.influencersNumToDislay = Math.min(scope.influencersNumToDislay, recordInfluencers.length); - let othersCount = Math.max(recordInfluencers.length - scope.influencersNumToDislay, 0); - - if (othersCount === 1) { - // Display the 1 extra influencer as displaying "and 1 more" would also take up a line. - scope.influencersNumToDislay++; - othersCount = 0; - } - - const influencers = []; - for (let i = 0; i < scope.influencersNumToDislay; i++) { - _.each(recordInfluencers[i], (influencerFieldValue, influencerFieldName) => { - influencers.push({ 'name': influencerFieldName, 'value': influencerFieldValue }); - }); - } - - scope.influencers = influencers; - scope.otherInfluencersCount = othersCount; - } - } - } - - - return { - restrict: 'AE', - replace: false, - scope: {}, - template, - link: link - }; -}); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/index.js b/x-pack/plugins/ml/public/components/anomalies_table/index.js index 65271895fb31..f9fb2bc0dafa 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/index.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/index.js @@ -5,7 +5,6 @@ */ - -import './styles/main.less'; -import './anomalies_table_directive.js'; +import './anomalies_table_directive'; import './anomalies_table_service.js'; +import './styles/main.less'; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell.js b/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell.js similarity index 57% rename from x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell.js rename to x-pack/plugins/ml/public/components/anomalies_table/influencers_cell.js index a712904b3029..78400b3fe7eb 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell.js +++ b/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell.js @@ -5,7 +5,6 @@ */ - import _ from 'lodash'; import PropTypes from 'prop-types'; import React, { Component } from 'react'; @@ -21,23 +20,8 @@ export class InfluencersCell extends Component { constructor(props) { super(props); - this.limit = props.limit; - const recordInfluencers = props.influencers || []; - this.influencers = []; - _.each(recordInfluencers, (influencer) => { - _.each(influencer, (influencerFieldValue, influencerFieldName) => { - this.influencers.push({ - influencerFieldName, - influencerFieldValue - }); - }); - }); - - // Allow one more influencer than the supplied limit as displaying - // 'and 1 more' would take up an extra line. - const showAll = this.influencers.length <= (this.limit + 1); this.state = { - showAll + showAll: false }; } @@ -45,36 +29,42 @@ export class InfluencersCell extends Component { this.setState({ showAll: !this.state.showAll }); } - renderInfluencers() { - const numberToDisplay = this.state.showAll === false ? this.limit : this.influencers.length; - const displayInfluencers = this.influencers.slice(0, numberToDisplay); + renderInfluencers(influencers) { + const numberToDisplay = (this.state.showAll === false) ? this.props.limit : influencers.length; + const displayInfluencers = influencers.slice(0, numberToDisplay); - this.othersCount = Math.max(this.influencers.length - numberToDisplay, 0); - if (this.othersCount === 1) { + let othersCount = Math.max(influencers.length - numberToDisplay, 0); + if (othersCount === 1) { // Display the additional influencer. - displayInfluencers.push(this.influencers[this.limit]); - this.othersCount = 0; + displayInfluencers.push(influencers[this.props.limit]); + othersCount = 0; } - return displayInfluencers.map((influencer, index) => { - return ( -
{influencer.influencerFieldName}: {influencer.influencerFieldValue}
- ); - }); + const displayRows = displayInfluencers.map((influencer, index) => ( +
{influencer.influencerFieldName}: {influencer.influencerFieldValue}
+ )); + + + return ( + + {displayRows} + {this.renderOthers(influencers.length, othersCount)} + + ); } - renderOthers() { - if (this.othersCount > 0) { + renderOthers(totalCount, othersCount) { + if (othersCount > 0) { return (
this.toggleAllInfluencers()} > - and {this.othersCount} more + and {othersCount} more
); - } else if (this.influencers.length > this.limit + 1) { + } else if (totalCount > this.props.limit + 1) { return (
{ + _.each(influencer, (influencerFieldValue, influencerFieldName) => { + influencers.push({ + influencerFieldName, + influencerFieldValue + }); + }); + }); + return (
- {this.renderInfluencers()} - {this.renderOthers()} + {this.renderInfluencers(influencers)} + {this.renderOthers(influencers)}
); } diff --git a/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell_directive.js b/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell_directive.js deleted file mode 100644 index 09aa97c28f34..000000000000 --- a/x-pack/plugins/ml/public/components/anomalies_table/influencers_cell/influencers_cell_directive.js +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - - - - -import 'ngreact'; - -import { InfluencersCell } from './influencers_cell'; - -import { uiModules } from 'ui/modules'; -const module = uiModules.get('apps/ml', ['react']); -module.directive('mlInfluencersCell', function (reactDirective) { - return reactDirective(InfluencersCell, undefined, { restrict: 'E' }); -}); diff --git a/x-pack/plugins/ml/public/components/anomalies_table/links_menu.js b/x-pack/plugins/ml/public/components/anomalies_table/links_menu.js new file mode 100644 index 000000000000..d77716049237 --- /dev/null +++ b/x-pack/plugins/ml/public/components/anomalies_table/links_menu.js @@ -0,0 +1,414 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +import _ from 'lodash'; +import moment from 'moment'; +import rison from 'rison-node'; +import PropTypes from 'prop-types'; +import React, { Component } from 'react'; + +import { + EuiButtonEmpty, + EuiContextMenuPanel, + EuiContextMenuItem, + EuiPopover +} from '@elastic/eui'; + +import 'ui/timefilter'; +import chrome from 'ui/chrome'; +import { toastNotifications } from 'ui/notify'; + +import { ES_FIELD_TYPES } from 'plugins/ml/../common/constants/field_types'; +import { parseInterval } from 'plugins/ml/../common/util/parse_interval'; +import { getFieldTypeFromMapping } from 'plugins/ml/services/mapping_service'; +import { ml } from 'plugins/ml/services/ml_api_service'; +import { mlJobService } from 'plugins/ml/services/job_service'; +import { getUrlForRecord } from 'plugins/ml/util/custom_url_utils'; +import { getIndexPatterns } from 'plugins/ml/util/index_utils'; +import { replaceStringTokens } from 'plugins/ml/util/string_utils'; + + +/* + * Component for rendering the links menu inside a cell in the anomalies table. + */ +export class LinksMenu extends Component { + constructor(props) { + super(props); + + this.state = { + isPopoverOpen: false, + toasts: [] + }; + } + + openCustomUrl = (customUrl) => { + const { anomaly, interval, isAggregatedData } = this.props; + + console.log('Anomalies Table - open customUrl for record:', anomaly); + + // If url_value contains $earliest$ and $latest$ tokens, add in times to the source record. + // Create a copy of the record as we are adding properties into it. + const record = _.cloneDeep(anomaly.source); + const timestamp = record.timestamp; + const configuredUrlValue = customUrl.url_value; + const timeRangeInterval = parseInterval(customUrl.time_range); + if (configuredUrlValue.includes('$earliest$')) { + let earliestMoment = moment(timestamp); + if (timeRangeInterval !== null) { + earliestMoment.subtract(timeRangeInterval); + } else { + earliestMoment = moment(timestamp).startOf(interval); + if (interval === 'hour') { + // Start from the previous hour. + earliestMoment.subtract(1, 'h'); + } + } + record.earliest = earliestMoment.toISOString(); // e.g. 2016-02-08T16:00:00.000Z + } + + if (configuredUrlValue.includes('$latest$')) { + let latestMoment = moment(timestamp).add(record.bucket_span, 's'); + if (timeRangeInterval !== null) { + latestMoment.add(timeRangeInterval); + } else { + if (isAggregatedData === true) { + latestMoment = moment(timestamp).endOf(interval); + if (interval === 'hour') { + // Show to the end of the next hour. + latestMoment.add(1, 'h'); // e.g. 2016-02-08T18:59:59.999Z + } + } + } + record.latest = latestMoment.toISOString(); + } + + // If url_value contains $mlcategoryterms$ or $mlcategoryregex$, add in the + // terms and regex for the selected categoryId to the source record. + if ((configuredUrlValue.includes('$mlcategoryterms$') || configuredUrlValue.includes('$mlcategoryregex$')) + && _.has(record, 'mlcategory')) { + const jobId = record.job_id; + + // mlcategory in the source record will be an array + // - use first value (will only ever be more than one if influenced by category other than by/partition/over). + const categoryId = record.mlcategory[0]; + + ml.results.getCategoryDefinition(jobId, categoryId) + .then((resp) => { + // Prefix each of the terms with '+' so that the Elasticsearch Query String query + // run in a drilldown Kibana dashboard has to match on all terms. + const termsArray = resp.terms.split(' ').map(term => `+${term}`); + record.mlcategoryterms = termsArray.join(' '); + record.mlcategoryregex = resp.regex; + + // Replace any tokens in the configured url_value with values from the source record, + // and then open link in a new tab/window. + const urlPath = replaceStringTokens(customUrl.url_value, record, true); + window.open(urlPath, '_blank'); + + }).catch((resp) => { + console.log('openCustomUrl(): error loading categoryDefinition:', resp); + toastNotifications.addDanger( + `Unable to open link as an error occurred loading details on category ID ${categoryId}`); + }); + + } else { + // Replace any tokens in the configured url_value with values from the source record, + // and then open link in a new tab/window. + const urlPath = getUrlForRecord(customUrl, record); + window.open(urlPath, '_blank'); + } + + }; + + viewSeries = () => { + const record = this.props.anomaly.source; + const bounds = this.props.timefilter.getActiveBounds(); + const from = bounds.min.toISOString(); // e.g. 2016-02-08T16:00:00.000Z + const to = bounds.max.toISOString(); + + // Zoom to show 50 buckets either side of the record. + const recordTime = moment(record.timestamp); + const zoomFrom = recordTime.subtract(50 * record.bucket_span, 's').toISOString(); + const zoomTo = recordTime.add(100 * record.bucket_span, 's').toISOString(); + + // Extract the by, over and partition fields for the record. + const entityCondition = {}; + + if (_.has(record, 'partition_field_value')) { + entityCondition[record.partition_field_name] = record.partition_field_value; + } + + if (_.has(record, 'over_field_value')) { + entityCondition[record.over_field_name] = record.over_field_value; + } + + if (_.has(record, 'by_field_value')) { + // Note that analyses with by and over fields, will have a top-level by_field_name, + // but the by_field_value(s) will be in the nested causes array. + // TODO - drilldown from cause in expanded row only? + entityCondition[record.by_field_name] = record.by_field_value; + } + + // Use rison to build the URL . + const _g = rison.encode({ + ml: { + jobIds: [record.job_id] + }, + refreshInterval: { + display: 'Off', + pause: false, + value: 0 + }, + time: { + from: from, + to: to, + mode: 'absolute' + } + }); + + const _a = rison.encode({ + mlTimeSeriesExplorer: { + zoom: { + from: zoomFrom, + to: zoomTo + }, + detectorIndex: record.detector_index, + entities: entityCondition, + }, + filters: [], + query: { + query_string: { + analyze_wildcard: true, + query: '*' + } + } + }); + + // Need to encode the _a parameter in case any entities contain unsafe characters such as '+'. + let path = `${chrome.getBasePath()}/app/ml#/timeseriesexplorer`; + path += `?_g=${_g}&_a=${encodeURIComponent(_a)}`; + window.open(path, '_blank'); + } + + viewExamples = () => { + const categoryId = this.props.anomaly.entityValue; + const record = this.props.anomaly.source; + const indexPatterns = getIndexPatterns(); + + const job = mlJobService.getJob(this.props.anomaly.jobId); + if (job === undefined) { + console.log(`viewExamples(): no job found with ID: ${this.props.anomaly.jobId}`); + toastNotifications.addDanger( + `Unable to view examples as no details could be found for job ID ${this.props.anomaly.jobId}`); + return; + } + const categorizationFieldName = job.analysis_config.categorization_field_name; + const datafeedIndices = job.datafeed_config.indices; + // Find the type of the categorization field i.e. text (preferred) or keyword. + // Uses the first matching field found in the list of indices in the datafeed_config. + // attempt to load the field type using each index. we have to do it this way as _field_caps + // doesn't specify which index a field came from unless there is a clash. + let i = 0; + findFieldType(datafeedIndices[i]); + + function findFieldType(index) { + getFieldTypeFromMapping(index, categorizationFieldName) + .then((resp) => { + if (resp !== '') { + createAndOpenUrl(index, resp); + } else { + i++; + if (i < datafeedIndices.length) { + findFieldType(datafeedIndices[i]); + } else { + error(); + } + } + }) + .catch(() => { + error(); + }); + } + + function createAndOpenUrl(index, categorizationFieldType) { + // Find the ID of the index pattern with a title attribute which matches the + // index configured in the datafeed. If a Kibana index pattern has not been created + // for this index, then the user will see a warning message on the Discover tab advising + // them that no matching index pattern has been configured. + let indexPatternId = index; + for (let j = 0; j < indexPatterns.length; j++) { + if (indexPatterns[j].get('title') === index) { + indexPatternId = indexPatterns[j].id; + break; + } + } + + // Get the definition of the category and use the terms or regex to view the + // matching events in the Kibana Discover tab depending on whether the + // categorization field is of mapping type text (preferred) or keyword. + ml.results.getCategoryDefinition(record.job_id, categoryId) + .then((resp) => { + let query = null; + // Build query using categorization regex (if keyword type) or terms (if text type). + // Check for terms or regex in case categoryId represents an anomaly from the absence of the + // categorization field in documents (usually indicated by a categoryId of -1). + if (categorizationFieldType === ES_FIELD_TYPES.KEYWORD) { + if (resp.regex) { + query = `${categorizationFieldName}:/${resp.regex}/`; + } + } else { + if (resp.terms) { + query = `${categorizationFieldName}:` + resp.terms.split(' ').join(` AND ${categorizationFieldName}:`); + } + } + + const recordTime = moment(record.timestamp); + const from = recordTime.toISOString(); + const to = recordTime.add(record.bucket_span, 's').toISOString(); + + // Use rison to build the URL . + const _g = rison.encode({ + refreshInterval: { + display: 'Off', + pause: false, + value: 0 + }, + time: { + from: from, + to: to, + mode: 'absolute' + } + }); + + const appStateProps = { + index: indexPatternId, + filters: [] + }; + if (query !== null) { + appStateProps.query = { + query_string: { + analyze_wildcard: true, + query: query + } + }; + } + const _a = rison.encode(appStateProps); + + // Need to encode the _a parameter as it will contain characters such as '+' if using the regex. + let path = chrome.getBasePath(); + path += '/app/kibana#/discover'; + path += '?_g=' + _g; + path += '&_a=' + encodeURIComponent(_a); + window.open(path, '_blank'); + + }).catch((resp) => { + console.log('viewExamples(): error loading categoryDefinition:', resp); + toastNotifications.addDanger( + `Unable to view examples as an error occurred loading details on category ID ${categoryId}`); + }); + + } + + function error() { + console.log(`viewExamples(): error finding type of field ${categorizationFieldName} in indices:`, + datafeedIndices); + toastNotifications.addDanger( + `Unable to view examples of documents with mlcategory ${categoryId} ` + + `as no mapping could be found for the categorization field ${categorizationFieldName}`); + } + }; + + onButtonClick = () => { + this.setState(prevState => ({ + isPopoverOpen: !prevState.isPopoverOpen, + })); + }; + + closePopover = () => { + this.setState({ + isPopoverOpen: false, + }); + }; + + render() { + const { anomaly, showViewSeriesLink } = this.props; + + const button = ( + + Open link + + ); + + const items = []; + if (anomaly.customUrls !== undefined) { + anomaly.customUrls.forEach((customUrl, index) => { + items.push( + { this.closePopover(); this.openCustomUrl(customUrl); }} + > + {customUrl.url_name} + + ); + }); + } + + if (showViewSeriesLink === true && anomaly.isTimeSeriesViewDetector === true) { + items.push( + { this.closePopover(); this.viewSeries(); }} + > + View series + + ); + } + + if (anomaly.entityName === 'mlcategory') { + items.push( + { this.closePopover(); this.viewExamples(); }} + > + View examples + + ); + } + + return ( + + + + ); + } +} + +LinksMenu.propTypes = { + anomaly: PropTypes.object.isRequired, + showViewSeriesLink: PropTypes.bool, + isAggregatedData: PropTypes.bool, + interval: PropTypes.string, + timefilter: PropTypes.object.isRequired +}; diff --git a/x-pack/plugins/ml/public/components/anomalies_table/styles/main.less b/x-pack/plugins/ml/public/components/anomalies_table/styles/main.less index 9527ce538d92..a71d790ce538 100644 --- a/x-pack/plugins/ml/public/components/anomalies_table/styles/main.less +++ b/x-pack/plugins/ml/public/components/anomalies_table/styles/main.less @@ -1,126 +1,114 @@ -ml-anomalies-table { - - .anomalies-table { - - padding: 10px; - - .no-results-item { - text-align: center; - padding-top: 10px; - } - - .table { - margin-bottom: 35px; - - td, th { - color: #2d2d2d; - font-size: 12px; - white-space: nowrap; - - .kuiButton.dropdown-toggle { - font-size: 12px; - } - - .fa.fa-arrow-up, .fa.fa-arrow-down { - font-size: 11px; - color: #686868; - } - - button { - border: none; - } - button:focus { - outline: none; - } - } - - td { - button { - font-family: inherit; - background-color: transparent; - } - } - - th { - button { - background-color: #FFFFFF; - padding: 0px 2px; - } - } - } - - .agg-table-paginated { - overflow-x: auto; - .dropdown-menu { - left: -60px; - } - } - - .ml-anomaly-interim-result { - font-style:italic; - padding-bottom: 3px; - } - - .ml-tablerow-expanded { - width: 100%; - padding: 5px 20px; - overflow: hidden; - font-size: 12px; - - table { - td { - padding: 0px 0px 2px 0px; - button { - padding: 0px 0px; - } - } - - tr>td:first-child { - padding-left: 2px; - vertical-align: top; - } - - td:first-child { - width: 140px; - } - - td:nth-child(2) { - padding-left: 5px; - } - } - - } - - .ml-tablerow-expanded-heading { - font-weight: bold; - display: block; - padding-top: 5px; - } - - .ml-tablerow-expanded-heading:first-child { - padding-top: 0px; - } - - .ml-tablerow-expanded-mv-description { - display: block; - } - +.ml-anomalies-table { + .ml-icon-severity-critical, + .ml-icon-severity-major, + .ml-icon-severity-minor, + .ml-icon-severity-warning, + .ml-icon-severity-unknown { + color: inherit; + text-shadow: none; } + + + .ml-icon-severity-critical { + .euiIcon { + fill: #fe5050; + } + } + + .ml-icon-severity-major { + .euiIcon { + fill: #fba740; + } + } + + .ml-icon-severity-minor { + .euiIcon { + fill: #fdec25; + } + } + + .ml-icon-severity-warning { + .euiIcon { + fill: #8bc8fb; + } + } + + .ml-icon-severity-unknown { + .euiIcon { + fill: #c0c0c0; + } + } + + tr th:first-child, + tr td:first-child { + width: 32px; + } + + .euiTableCellContent { + .euiHealth { + font-size: inherit; + } + + .filter-button { + opacity: 0.3; + width: 20px; + padding-top: 2px; + + .euiIcon { + width: 14px; + height: 14px; + } + } + + .filter-button:hover { + opacity: 1; + } + } + + .euiContextMenuItem { + min-width: 150px + } + + .category-example { + display: block; + white-space: nowrap; + } + + .interim-result { + font-style: italic; + } + + .ml-anomalies-table-details { + padding: 4px 32px; + max-height: 1000px; + overflow-y: auto; + + .anomaly-description-list { + + .euiDescriptionList__title { + margin-top: 0px; + flex-basis: 15%; + font-size: inherit; + } + + .euiDescriptionList__description { + margin-top: 0px; + flex-basis: 85%; + font-size: inherit; + } + + .filter-button { + height: 20px; + padding-top: 2px; + + .euiButtonIcon__icon { + -webkit-transform: translateY(-7px); + transform: translateY(-7px); + } + } + + } + } + } -.ml-anomalies-table.dropdown-menu { - min-width: 120px; - font-size: 12px; -} - -.ml-anomalies-table.dropdown-menu > li > a { - color: #444444; - text-decoration: none; -} - -.ml-anomalies-table.dropdown-menu > li > a:hover, -.ml-anomalies-table.dropdown-menu > li > a:active, -.ml-anomalies-table.dropdown-menu > li > a:focus { - color: #ffffff; - box-shadow: none; -} diff --git a/x-pack/plugins/ml/public/explorer/explorer.html b/x-pack/plugins/ml/public/explorer/explorer.html index 5f34772052e1..9c6d3cb41ac6 100644 --- a/x-pack/plugins/ml/public/explorer/explorer.html +++ b/x-pack/plugins/ml/public/explorer/explorer.html @@ -141,13 +141,10 @@
-
- - -
+ + diff --git a/x-pack/plugins/ml/public/explorer/explorer_controller.js b/x-pack/plugins/ml/public/explorer/explorer_controller.js index 8fbff648456d..88491a11d8bc 100644 --- a/x-pack/plugins/ml/public/explorer/explorer_controller.js +++ b/x-pack/plugins/ml/public/explorer/explorer_controller.js @@ -18,6 +18,7 @@ import DragSelect from 'dragselect'; import moment from 'moment'; import 'plugins/ml/components/anomalies_table'; +import 'plugins/ml/components/controls'; import 'plugins/ml/components/influencers_list'; import 'plugins/ml/components/job_select_list'; @@ -32,10 +33,12 @@ import { checkGetJobsPrivilege } from 'plugins/ml/privilege/check_privilege'; import { loadIndexPatterns, getIndexPatterns } from 'plugins/ml/util/index_utils'; import { refreshIntervalWatcher } from 'plugins/ml/util/refresh_interval_watcher'; import { IntervalHelperProvider, getBoundsRoundedToInterval } from 'plugins/ml/util/ml_time_buckets'; +import { ml } from 'plugins/ml/services/ml_api_service'; import { mlResultsService } from 'plugins/ml/services/results_service'; import { mlJobService } from 'plugins/ml/services/job_service'; import { mlFieldFormatService } from 'plugins/ml/services/field_format_service'; import { JobSelectServiceProvider } from 'plugins/ml/components/job_select_list/job_select_service'; +import { isTimeSeriesViewDetector } from 'plugins/ml/../common/util/job_utils'; uiRoutes .when('/explorer/?', { @@ -60,6 +63,7 @@ module.controller('MlExplorerController', function ( mlCheckboxShowChartsService, mlExplorerDashboardService, mlSelectLimitService, + mlSelectIntervalService, mlSelectSeverityService) { $scope.timeFieldName = 'timestamp'; @@ -75,12 +79,12 @@ module.controller('MlExplorerController', function ( const $mlExplorer = $('.ml-explorer'); const MAX_INFLUENCER_FIELD_VALUES = 10; + const MAX_CATEGORY_EXAMPLES = 10; const VIEW_BY_JOB_LABEL = 'job ID'; const ALLOW_CELL_RANGE_SELECTION = mlExplorerDashboardService.allowCellRangeSelection; let disableDragSelectOnMouseLeave = true; $scope.queryFilters = []; - $scope.anomalyRecords = []; const dragSelect = new DragSelect({ selectables: document.querySelectorAll('.sl-cell'), @@ -309,18 +313,36 @@ module.controller('MlExplorerController', function ( // Returns the time range of the cell(s) currently selected in the swimlane. // If no cell(s) are currently selected, returns the dashboard time range. const bounds = timefilter.getActiveBounds(); - - // time property of the cell data is an array, with the elements being - // the start times of the first and last cell selected. - const earliestMs = cellData.time[0] !== undefined ? ((cellData.time[0]) * 1000) : bounds.min.valueOf(); + let earliestMs = bounds.min.valueOf(); let latestMs = bounds.max.valueOf(); - if (cellData.time[1] !== undefined) { - // Subtract 1 ms so search does not include start of next bucket. - latestMs = ((cellData.time[1] + cellData.interval) * 1000) - 1; + + if (cellData !== undefined && cellData.time !== undefined) { + // time property of the cell data is an array, with the elements being + // the start times of the first and last cell selected. + earliestMs = (cellData.time[0] !== undefined) ? cellData.time[0] * 1000 : bounds.min.valueOf(); + latestMs = bounds.max.valueOf(); + if (cellData.time[1] !== undefined) { + // Subtract 1 ms so search does not include start of next bucket. + latestMs = ((cellData.time[1] + cellData.interval) * 1000) - 1; + } } + return { earliestMs, latestMs }; } + function getSelectionInfluencers(cellData) { + const influencers = []; + + if (cellData !== undefined && cellData.fieldName !== undefined && + cellData.fieldName !== VIEW_BY_JOB_LABEL) { + cellData.laneLabels.forEach((laneLabel) =>{ + influencers.push({ fieldName: $scope.swimlaneViewByFieldName, fieldValue: laneLabel }); + }); + } + + return influencers; + } + // Listener for click events in the swimlane and load corresponding anomaly data. // Empty cellData is passed on clicking outside a cell with score > 0. const swimlaneCellClickListener = function (cellData) { @@ -332,8 +354,6 @@ module.controller('MlExplorerController', function ( } clearSelectedAnomalies(); } else { - let jobIds = []; - const influencers = []; const timerange = getSelectionTimeRange(cellData); if (cellData.fieldName === undefined) { @@ -343,22 +363,15 @@ module.controller('MlExplorerController', function ( $scope.viewByLoadedForTimeFormatted = moment(timerange.earliestMs).format('MMMM Do YYYY, HH:mm'); } - if (cellData.fieldName === VIEW_BY_JOB_LABEL) { - jobIds = cellData.laneLabels; - } else { - jobIds = $scope.getSelectedJobIds(); - - if (cellData.fieldName !== undefined) { - cellData.laneLabels.forEach((laneLabel) =>{ - influencers.push({ fieldName: $scope.swimlaneViewByFieldName, fieldValue: laneLabel }); - }); - } - } + const jobIds = (cellData.fieldName === VIEW_BY_JOB_LABEL) ? + cellData.laneLabels : $scope.getSelectedJobIds(); + const influencers = getSelectionInfluencers(cellData); $scope.cellData = cellData; + loadAnomaliesTableData(); + const args = [jobIds, influencers, timerange.earliestMs, timerange.latestMs]; - loadAnomalies(...args); - $scope.loadAnomaliesForCharts(...args); + loadDataForCharts(...args); } }; mlExplorerDashboardService.swimlaneCellClick.watch(swimlaneCellClickListener); @@ -387,6 +400,12 @@ module.controller('MlExplorerController', function ( }; mlSelectSeverityService.state.watch(anomalyChartsSeverityListener); + const tableControlsListener = function () { + loadAnomaliesTableData(); + }; + mlSelectIntervalService.state.watch(tableControlsListener); + mlSelectSeverityService.state.watch(tableControlsListener); + const swimlaneLimitListener = function () { loadViewBySwimlane([]); clearSelectedAnomalies(); @@ -405,42 +424,37 @@ module.controller('MlExplorerController', function ( mlExplorerDashboardService.swimlaneCellClick.unwatch(swimlaneCellClickListener); mlExplorerDashboardService.swimlaneRenderDone.unwatch(swimlaneRenderDoneListener); mlSelectSeverityService.state.unwatch(anomalyChartsSeverityListener); + mlSelectIntervalService.state.unwatch(tableControlsListener); + mlSelectSeverityService.state.unwatch(tableControlsListener); mlSelectLimitService.state.unwatch(swimlaneLimitListener); - $scope.cellData = undefined; + delete $scope.cellData; refreshWatcher.cancel(); // Cancel listening for updates to the global nav state. navListener(); }); - $scope.loadAnomaliesForCharts = function (jobIds, influencers, earliestMs, latestMs) { - // Load the top anomalies (by record_score) which will be displayed in the charts. - // TODO - combine this with loadAnomalies(). - mlResultsService.getRecordsForInfluencer( - jobIds, influencers, 0, earliestMs, latestMs, 500 - ).then((resp) => { - $scope.anomalyChartRecords = resp.records; - console.log('Explorer anomaly charts data set:', $scope.anomalyChartRecords); - - if (mlCheckboxShowChartsService.state.get('showCharts')) { - mlExplorerDashboardService.anomalyDataChange.changed( - $scope.anomalyChartRecords, earliestMs, latestMs - ); - } - }); - }; - - function loadAnomalies(jobIds, influencers, earliestMs, latestMs) { - // Loads the anomalies for the table, plus the scores for - // the Top Influencers List for the influencers in the anomaly records. - + function loadDataForCharts(jobIds, influencers, earliestMs, latestMs) { + // Loads the data used to populate the anomaly charts and the Top Influencers List. if (influencers.length === 0) { getTopInfluencers(jobIds, earliestMs, latestMs); } + // Load the top anomalies (by record_score) which will be displayed in the charts. mlResultsService.getRecordsForInfluencer( jobIds, influencers, 0, earliestMs, latestMs, 500 ) .then((resp) => { + if ($scope.cellData !== undefined && _.keys($scope.cellData).length > 0) { + $scope.anomalyChartRecords = resp.records; + console.log('Explorer anomaly charts data set:', $scope.anomalyChartRecords); + + if (mlCheckboxShowChartsService.state.get('showCharts')) { + mlExplorerDashboardService.anomalyDataChange.changed( + $scope.anomalyChartRecords, earliestMs, latestMs + ); + } + } + if (influencers.length > 0) { // Filter the Top Influencers list to show just the influencers from // the records in the selected time range. @@ -483,13 +497,6 @@ module.controller('MlExplorerController', function ( getTopInfluencers(jobIds, earliestMs, latestMs, filterInfluencers); } - - // Use $evalAsync to ensure the update happens after the child scope is updated with the new data. - $scope.$evalAsync(() => { - // Sort in descending time order before storing in scope. - $scope.anomalyRecords = _.chain(resp.records).sortBy(record => record[$scope.timeFieldName]).reverse().value(); - console.log('Explorer anomalies table data set:', $scope.anomalyRecords); - }); }); } @@ -767,10 +774,62 @@ module.controller('MlExplorerController', function ( } } + function loadAnomaliesTableData() { + const cellData = $scope.cellData; + const jobIds = ($scope.cellData !== undefined && cellData.fieldName === VIEW_BY_JOB_LABEL) ? + cellData.laneLabels : $scope.getSelectedJobIds(); + const influencers = getSelectionInfluencers(cellData); + const timeRange = getSelectionTimeRange(cellData); + + ml.results.getAnomaliesTableData( + jobIds, + [], + influencers, + mlSelectIntervalService.state.get('interval').val, + mlSelectSeverityService.state.get('threshold').val, + timeRange.earliestMs, + timeRange.latestMs, + 500, + MAX_CATEGORY_EXAMPLES + ).then((resp) => { + const anomalies = resp.anomalies; + const detectorsByJob = mlJobService.detectorsByJob; + anomalies.forEach((anomaly) => { + // Add a detector property to each anomaly. + // Default to functionDescription if no description available. + // TODO - when job_service is moved server_side, move this to server endpoint. + const jobId = anomaly.jobId; + anomaly.detector = _.get(detectorsByJob, + [jobId, anomaly.detectorIndex, 'detector_description'], + anomaly.source.function_description); + + // Add properties used for building the links menu. + // TODO - when job_service is moved server_side, move this to server endpoint. + anomaly.isTimeSeriesViewDetector = isTimeSeriesViewDetector( + mlJobService.getJob(jobId), anomaly.detectorIndex); + if (_.has(mlJobService.customUrlsByJob, jobId)) { + anomaly.customUrls = mlJobService.customUrlsByJob[jobId]; + } + }); + + $scope.$evalAsync(() => { + $scope.tableData = { + anomalies, + interval: resp.interval, + examplesByJobId: resp.examplesByJobId, + showViewSeriesLink: true + }; + }); + + }).catch((resp) => { + console.log('Explorer - error loading data for anomalies table:', resp); + }); + } + function clearSelectedAnomalies() { $scope.anomalyChartRecords = []; - $scope.anomalyRecords = []; $scope.viewByLoadedForTimeFormatted = null; + delete $scope.cellData; // With no swimlane selection, display anomalies over all time in the table. const jobIds = $scope.getSelectedJobIds(); @@ -778,7 +837,8 @@ module.controller('MlExplorerController', function ( const earliestMs = bounds.min.valueOf(); const latestMs = bounds.max.valueOf(); mlExplorerDashboardService.anomalyDataChange.changed($scope.anomalyChartRecords, earliestMs, latestMs); - loadAnomalies(jobIds, [], earliestMs, latestMs); + loadDataForCharts(jobIds, [], earliestMs, latestMs); + loadAnomaliesTableData(); } function calculateSwimlaneBucketInterval() { diff --git a/x-pack/plugins/ml/public/explorer/styles/main.less b/x-pack/plugins/ml/public/explorer/styles/main.less index 16acf6d2f363..534b8cc73a6f 100644 --- a/x-pack/plugins/ml/public/explorer/styles/main.less +++ b/x-pack/plugins/ml/public/explorer/styles/main.less @@ -2,7 +2,6 @@ width: 100%; display: inline-block; color: #555; - font-size: 10px; .visualize-error { h4 { @@ -333,3 +332,5 @@ } } + + diff --git a/x-pack/plugins/ml/public/services/ml_api_service/results.js b/x-pack/plugins/ml/public/services/ml_api_service/results.js index a405eacecdeb..5f7b76ce2686 100644 --- a/x-pack/plugins/ml/public/services/ml_api_service/results.js +++ b/x-pack/plugins/ml/public/services/ml_api_service/results.js @@ -15,6 +15,7 @@ const basePath = chrome.addBasePath('/api/ml'); export const results = { getAnomaliesTableData( jobIds, + criteriaFields, influencers, aggregationInterval, threshold, @@ -28,6 +29,7 @@ export const results = { method: 'POST', data: { jobIds, + criteriaFields, influencers, aggregationInterval, threshold, diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js index 681cda5a6b14..93f7797cfc9e 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseries_chart_directive.js @@ -28,6 +28,7 @@ import { numTicksForDateFormat } from 'plugins/ml/util/chart_utils'; import { TimeBuckets } from 'ui/time_buckets'; +import { mlAnomaliesTableService } from 'plugins/ml/components/anomalies_table/anomalies_table_service'; import ContextChartMask from 'plugins/ml/timeseriesexplorer/context_chart_mask'; import { findNearestChartPointToTime } from 'plugins/ml/timeseriesexplorer/timeseriesexplorer_utils'; import { mlEscape } from 'plugins/ml/util/string_utils'; @@ -40,7 +41,6 @@ module.directive('mlTimeseriesChart', function ( $compile, $timeout, timefilter, - mlAnomaliesTableService, Private, mlChartTooltipService) { diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer.html b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer.html index 03b078b7535d..a3549c52a0df 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer.html +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer.html @@ -127,10 +127,9 @@ - + table-data="tableData" + filter="filter" + /> diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js index 1a15874f73f5..19b51da3d65e 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_controller.js @@ -16,6 +16,7 @@ import _ from 'lodash'; import moment from 'moment'; import 'plugins/ml/components/anomalies_table'; +import 'plugins/ml/components/controls'; import { notify } from 'ui/notify'; import uiRoutes from 'ui/routes'; @@ -41,6 +42,7 @@ import { IntervalHelperProvider, getBoundsRoundedToInterval } from 'plugins/ml/u import { mlResultsService } from 'plugins/ml/services/results_service'; import template from './timeseriesexplorer.html'; import { getMlNodeCount } from 'plugins/ml/ml_nodes_check/check_ml_nodes'; +import { ml } from 'plugins/ml/services/ml_api_service'; import { mlJobService } from 'plugins/ml/services/job_service'; import { mlFieldFormatService } from 'plugins/ml/services/field_format_service'; import { JobSelectServiceProvider } from 'plugins/ml/components/job_select_list/job_select_service'; @@ -70,7 +72,8 @@ module.controller('MlTimeSeriesExplorerController', function ( Private, timefilter, AppState, - mlAnomaliesTableService) { + mlSelectIntervalService, + mlSelectSeverityService) { $scope.timeFieldName = 'timestamp'; timefilter.enableTimeRangeSelector(); @@ -341,7 +344,7 @@ module.controller('MlTimeSeriesExplorerController', function ( $scope.refreshFocusData = function (fromDate, toDate) { - // Counter to keep track of what data sets have been loaded. + // Counter to keep track of the queries to populate the chart. let awaitingCount = 3; // This object is used to store the results of individual remote requests @@ -356,7 +359,6 @@ module.controller('MlTimeSeriesExplorerController', function ( awaitingCount--; if (awaitingCount === 0) { // Tell the results container directives to render the focus chart. - // Need to use $timeout to ensure the broadcast happens after the child scope is updated with the new data. refreshFocusData.focusChartData = processDataForFocusAnomalies( refreshFocusData.focusChartData, refreshFocusData.anomalyRecords, @@ -366,7 +368,8 @@ module.controller('MlTimeSeriesExplorerController', function ( refreshFocusData.focusChartData, refreshFocusData.scheduledEvents); - // All the data is ready now for a scope update + // All the data is ready now for a scope update. + // Use $evalAsync to ensure the update happens after the child scope is updated with the new data. $scope.$evalAsync(() => { $scope = Object.assign($scope, refreshFocusData); console.log('Time series explorer focus chart data set:', $scope.focusChartData); @@ -405,7 +408,7 @@ module.controller('MlTimeSeriesExplorerController', function ( console.log('Time series explorer - error getting metric data from elasticsearch:', resp); }); - // Query 2 - load records across selected time range. + // Query 2 - load all the records across selected time range for the chart anomaly markers. mlResultsService.getRecordsForCriteria( [$scope.selectedJob.job_id], $scope.criteriaFields, @@ -467,6 +470,9 @@ module.controller('MlTimeSeriesExplorerController', function ( }); } + // Load the data for the anomalies table. + loadAnomaliesTableData(searchBounds.min.valueOf(), searchBounds.max.valueOf()); + }; $scope.saveSeriesPropertiesAndRefresh = function () { @@ -480,6 +486,19 @@ module.controller('MlTimeSeriesExplorerController', function ( $scope.refresh(); }; + $scope.filter = function (field, value, operator) { + const entity = _.find($scope.entities, { fieldName: field }); + if (entity !== undefined) { + if (operator === '+' && entity.fieldValue !== value) { + entity.fieldValue = value; + $scope.saveSeriesPropertiesAndRefresh(); + } else if (operator === '-' && entity.fieldValue === value) { + entity.fieldValue = ''; + $scope.saveSeriesPropertiesAndRefresh(); + } + } + }; + $scope.loadForForecastId = function (forecastId) { mlForecastService.getForecastDateRange( $scope.selectedJob, @@ -548,28 +567,23 @@ module.controller('MlTimeSeriesExplorerController', function ( $scope.refresh(); }); - // Add a listener for filter changes triggered from the anomalies table. - const filterChangeListener = function (field, value, operator) { - const entity = _.find($scope.entities, { fieldName: field }); - if (entity !== undefined) { - if (operator === '+' && entity.fieldValue !== value) { - entity.fieldValue = value; - $scope.saveSeriesPropertiesAndRefresh(); - } else if (operator === '-' && entity.fieldValue === value) { - entity.fieldValue = ''; - $scope.saveSeriesPropertiesAndRefresh(); - } + // Reload the anomalies table if the Interval or Threshold controls are changed. + const tableControlsListener = function () { + if ($scope.zoomFrom !== undefined && $scope.zoomTo !== undefined) { + loadAnomaliesTableData($scope.zoomFrom.getTime(), $scope.zoomTo.getTime()); } }; + mlSelectIntervalService.state.watch(tableControlsListener); + mlSelectSeverityService.state.watch(tableControlsListener); - mlAnomaliesTableService.filterChange.watch(filterChangeListener); $scope.$on('$destroy', () => { refreshWatcher.cancel(); - mlAnomaliesTableService.filterChange.unwatch(filterChangeListener); + mlSelectIntervalService.state.unwatch(tableControlsListener); + mlSelectSeverityService.state.unwatch(tableControlsListener); }); - // When inside a dashboard in the ML plugin, listen for changes to job selection. + // Listen for changes to job selection. mlJobSelectService.listenJobSelectionChange($scope, (event, selections) => { // Clear the detectorIndex, entities and forecast info. if (selections.length > 0) { @@ -665,6 +679,49 @@ module.controller('MlTimeSeriesExplorerController', function ( }); } + function loadAnomaliesTableData(earliestMs, latestMs) { + ml.results.getAnomaliesTableData( + [$scope.selectedJob.job_id], + $scope.criteriaFields, + [], + mlSelectIntervalService.state.get('interval').val, + mlSelectSeverityService.state.get('threshold').val, + earliestMs, + latestMs, + ANOMALIES_MAX_RESULTS + ).then((resp) => { + const anomalies = resp.anomalies; + const detectorsByJob = mlJobService.detectorsByJob; + anomalies.forEach((anomaly) => { + // Add a detector property to each anomaly. + // Default to functionDescription if no description available. + // TODO - when job_service is moved server_side, move this to server endpoint. + const jobId = anomaly.jobId; + anomaly.detector = _.get(detectorsByJob, + [jobId, anomaly.detectorIndex, 'detector_description'], + anomaly.source.function_description); + + // Add properties used for building the links menu. + // TODO - when job_service is moved server_side, move this to server endpoint. + if (_.has(mlJobService.customUrlsByJob, jobId)) { + anomaly.customUrls = mlJobService.customUrlsByJob[jobId]; + } + }); + + $scope.$evalAsync(() => { + $scope.tableData = { + anomalies, + interval: resp.interval, + examplesByJobId: resp.examplesByJobId, + showViewSeriesLink: false + }; + }); + + }).catch((resp) => { + console.log('Time series explorer - error loading data for anomalies table:', resp); + }); + } + function updateControlsForDetector() { // Update the entity dropdown control(s) according to the partitioning fields for the selected detector. const detectorIndex = +$scope.detectorId; diff --git a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_utils.js b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_utils.js index a4761d172ac0..c59d5e1c8604 100644 --- a/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_utils.js +++ b/x-pack/plugins/ml/public/timeseriesexplorer/timeseriesexplorer_utils.js @@ -107,9 +107,9 @@ export function processDataForFocusAnomalies( const recordTime = record[timeFieldName]; let chartPoint = findNearestChartPointToTime(chartData, recordTime); - // TODO - handle case where there is an anomaly due to the absense of data + // TODO - handle case where there is an anomaly due to the absence of data // and there is no model plot. - if (chartPoint === undefined && chartData.length) { + if (chartPoint === undefined && chartData !== undefined && chartData.length) { // In case there is a record with a time after that of the last chart point, set the score // for the last chart point to that of the last record, if that record has a higher score. const lastChartPoint = chartData[chartData.length - 1]; @@ -167,6 +167,10 @@ export function processScheduledEventsForChart(chartData, scheduledEvents) { export function findNearestChartPointToTime(chartData, time) { let chartPoint; + if(chartData === undefined) { + return chartPoint; + } + for (let i = 0; i < chartData.length; i++) { if (chartData[i].date.getTime() === time) { chartPoint = chartData[i]; diff --git a/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js b/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js index 56094b7832ce..e1755f136dec 100644 --- a/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js +++ b/x-pack/plugins/ml/server/models/results_service/build_anomaly_table_items.js @@ -36,10 +36,14 @@ export function buildAnomalyTableItems(anomalyRecords, aggregationInterval) { // Fill out the remaining properties in each display record // for the columns to be displayed in the table. - return displayRecords.map((record) => { + const time = (new Date()).getTime(); + return displayRecords.map((record, index) => { const source = record.source; const jobId = source.job_id; + // Identify each row with a unique ID which is used by the table for row expansion. + record.rowId = `${time}_${index}`; + record.jobId = jobId; record.detectorIndex = source.detector_index; record.severity = source.record_score; @@ -64,29 +68,48 @@ export function buildAnomalyTableItems(anomalyRecords, aggregationInterval) { record.influencers = influencers; } + // Add fields to the display records for the actual and typical values. + // To ensure sorting in the EuiTable works correctly, add extra 'sort' + // properties which are single numeric values rather than the underlying arrays. + // These properties can be removed if EuiTable sorting logic can be customized + // - see https://github.com/elastic/eui/issues/425 const functionDescription = source.function_description || ''; const causes = source.causes || []; if (showActualForFunction(functionDescription) === true) { if (source.actual !== undefined) { record.actual = source.actual; + record.actualSort = getMetricSortValue(source.actual); } else { // If only a single cause, copy values to the top level. if (causes.length === 1) { record.actual = causes[0].actual; + record.actualSort = getMetricSortValue(causes[0].actual); } } } if (showTypicalForFunction(functionDescription) === true) { if (source.typical !== undefined) { record.typical = source.typical; + record.typicalSort = getMetricSortValue(source.typical); } else { // If only a single cause, copy values to the top level. if (causes.length === 1) { record.typical = causes[0].typical; + record.typicalSort = getMetricSortValue(causes[0].typical); } } } + // Add a sortable property for the magnitude of the factor by + // which the actual value is different from the typical. + if (Array.isArray(record.actual) && record.actual.length === 1 && + Array.isArray(record.typical) && record.typical.length === 1) { + const actualVal = Number(record.actual[0]); + const typicalVal = Number(record.typical[0]); + record.metricDescriptionSort = (actualVal > typicalVal) ? + actualVal / typicalVal : typicalVal / actualVal; + } + return record; }); @@ -160,3 +183,10 @@ function aggregateAnomalies(anomalyRecords, interval) { return summaryRecords; } + +function getMetricSortValue(value) { + // Returns a sortable value for a metric field (actual and typical values) + // from the supplied value, which for metric functions will be a single + // valued array. + return (Array.isArray(value) && value.length > 0) ? value[0] : value; +} diff --git a/x-pack/plugins/ml/server/models/results_service/results_service.js b/x-pack/plugins/ml/server/models/results_service/results_service.js index 07ddcf983b8b..7b569c26009a 100644 --- a/x-pack/plugins/ml/server/models/results_service/results_service.js +++ b/x-pack/plugins/ml/server/models/results_service/results_service.js @@ -17,6 +17,7 @@ import { ML_RESULTS_INDEX_PATTERN } from '../../../common/constants/index_patter // ML Results dashboards. const DEFAULT_QUERY_SIZE = 500; +const DEFAULT_MAX_EXAMPLES = 500; export function resultsServiceProvider(callWithRequest) { @@ -28,13 +29,14 @@ export function resultsServiceProvider(callWithRequest) { // anomalies are categorization anomalies in mlcategory. async function getAnomaliesTableData( jobIds, + criteriaFields, influencers, aggregationInterval, threshold, earliestMs, latestMs, - maxRecords, - maxExamples) { + maxRecords = DEFAULT_QUERY_SIZE, + maxExamples = DEFAULT_MAX_EXAMPLES) { // Build the query to return the matching anomaly record results. // Add criteria for the time range, record score, plus any specified job IDs. @@ -74,6 +76,15 @@ export function resultsServiceProvider(callWithRequest) { }); } + // Add in term queries for each of the specified criteria. + criteriaFields.forEach((criteria) => { + boolCriteria.push({ + term: { + [criteria.fieldName]: criteria.fieldValue + } + }); + }); + // Add a nested query to filter for each of the specified influencers. if (influencers.length > 0) { boolCriteria.push({ @@ -108,7 +119,7 @@ export function resultsServiceProvider(callWithRequest) { const resp = await callWithRequest('search', { index: ML_RESULTS_INDEX_PATTERN, - size: maxRecords !== undefined ? maxRecords : DEFAULT_QUERY_SIZE, + size: maxRecords, body: { query: { bool: { diff --git a/x-pack/plugins/ml/server/routes/results_service.js b/x-pack/plugins/ml/server/routes/results_service.js index efce4f9c50bf..07f261a07e0f 100644 --- a/x-pack/plugins/ml/server/routes/results_service.js +++ b/x-pack/plugins/ml/server/routes/results_service.js @@ -15,6 +15,7 @@ function getAnomaliesTableData(callWithRequest, payload) { const rs = resultsServiceProvider(callWithRequest); const { jobIds, + criteriaFields, influencers, aggregationInterval, threshold, @@ -24,6 +25,7 @@ function getAnomaliesTableData(callWithRequest, payload) { maxExamples } = payload; return rs.getAnomaliesTableData( jobIds, + criteriaFields, influencers, aggregationInterval, threshold,