From 6f464ad0d285c0ccb682d976a74c2db1aeff0b3a Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Thu, 31 Oct 2019 13:02:21 -0400 Subject: [PATCH] [ML] DataFrame Analytics: Regression results view (#49667) * enable analytics table view link for regression jobs * add results table component * can filter for training/testing * add search functionality to table * move shared types to analytics types file * anchor isTraining,predicted,actual columns to left. * ensure search string persists in search bar input * show jobStatus badge in results view * add jobType, status badge to outlier exploration page * update exploration tests --- .../public/data_frame_analytics/_index.scss | 1 + .../data_frame_analytics/common/analytics.ts | 19 + .../data_frame_analytics/common/fields.ts | 140 +++++ .../data_frame_analytics/common/index.ts | 6 + .../exploration/exploration.test.tsx | 5 +- .../components/exploration/exploration.tsx | 28 +- .../exploration/use_explore_data.ts | 11 +- .../regression_exploration/_index.scss | 1 + .../_regression_exploration.scss | 3 + .../regression_exploration/evaluate_panel.tsx | 62 ++- .../regression_exploration.tsx | 56 +- .../regression_exploration/results_table.tsx | 482 ++++++++++++++++++ .../use_explore_data.ts | 163 ++++++ .../pages/analytics_exploration/directive.tsx | 3 +- .../pages/analytics_exploration/page.tsx | 14 +- .../components/analytics_list/actions.tsx | 9 +- .../components/analytics_list/common.ts | 16 +- 17 files changed, 945 insertions(+), 74 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_index.scss create mode 100644 x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_regression_exploration.scss create mode 100644 x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx create mode 100644 x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/_index.scss b/x-pack/legacy/plugins/ml/public/data_frame_analytics/_index.scss index 4c0ecd8f9ce4..c231c405b536 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/_index.scss +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/_index.scss @@ -1,4 +1,5 @@ @import 'pages/analytics_exploration/components/exploration/index'; +@import 'pages/analytics_exploration/components/regression_exploration/index'; @import 'pages/analytics_management/components/analytics_list/index'; @import 'pages/analytics_management/components/create_analytics_form/index'; @import 'pages/analytics_management/components/create_analytics_flyout/index'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/analytics.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/analytics.ts index f99f9661f12e..385d50215cd2 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/analytics.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/analytics.ts @@ -29,6 +29,15 @@ interface RegressionAnalysis { }; } +export const SEARCH_SIZE = 1000; + +export enum INDEX_STATUS { + UNUSED, + LOADING, + LOADED, + ERROR, +} + export interface Eval { meanSquaredError: number | ''; rSquared: number | ''; @@ -91,6 +100,16 @@ export const getPredictionFieldName = (analysis: AnalysisConfig) => { return predictionFieldName; }; +export const getPredictedFieldName = (resultsField: string, analysis: AnalysisConfig) => { + // default is 'ml' + const predictionFieldName = getPredictionFieldName(analysis); + const defaultPredictionField = `${getDependentVar(analysis)}_prediction`; + const predictedField = `${resultsField}.${ + predictionFieldName ? predictionFieldName : defaultPredictionField + }`; + return predictedField; +}; + export const isOutlierAnalysis = (arg: any): arg is OutlierAnalysis => { const keys = Object.keys(arg); return keys.length === 1 && keys[0] === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/fields.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/fields.ts index 4ae3e8513e5b..b25f4b6ad8b9 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/fields.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/fields.ts @@ -5,6 +5,7 @@ */ import { getNestedProperty } from '../../util/object_utils'; +import { DataFrameAnalyticsConfig, getPredictedFieldName, getDependentVar } from './analytics'; export type EsId = string; export type EsDocSource = Record; @@ -16,6 +17,7 @@ export interface EsDoc extends Record { } export const MAX_COLUMNS = 20; +export const DEFAULT_REGRESSION_COLUMNS = 8; const ML__ID_COPY = 'ml__id_copy'; @@ -68,6 +70,104 @@ export const sortColumns = (obj: EsDocSource, resultsField: string) => (a: strin return a.localeCompare(b); }; +export const sortRegressionResultsFields = ( + a: string, + b: string, + jobConfig: DataFrameAnalyticsConfig +) => { + const dependentVariable = getDependentVar(jobConfig.analysis); + const resultsField = jobConfig.dest.results_field; + const predictedField = getPredictedFieldName(resultsField, jobConfig.analysis); + if (a === `${resultsField}.is_training`) { + return -1; + } + if (b === `${resultsField}.is_training`) { + return 1; + } + if (a === predictedField) { + return -1; + } + if (b === predictedField) { + return 1; + } + if (a === dependentVariable) { + return -1; + } + if (b === dependentVariable) { + return 1; + } + return a.localeCompare(b); +}; + +// Used to sort columns: +// Anchor on the left ml.is_training, , +export const sortRegressionResultsColumns = ( + obj: EsDocSource, + jobConfig: DataFrameAnalyticsConfig +) => (a: string, b: string) => { + const dependentVariable = getDependentVar(jobConfig.analysis); + const resultsField = jobConfig.dest.results_field; + const predictedField = getPredictedFieldName(resultsField, jobConfig.analysis); + + const typeofA = typeof obj[a]; + const typeofB = typeof obj[b]; + + if (a === `${resultsField}.is_training`) { + return -1; + } + + if (b === `${resultsField}.is_training`) { + return 1; + } + + if (a === predictedField) { + return -1; + } + + if (b === predictedField) { + return 1; + } + + if (a === dependentVariable) { + return -1; + } + + if (b === dependentVariable) { + return 1; + } + + if (typeofA !== 'string' && typeofB === 'string') { + return 1; + } + if (typeofA === 'string' && typeofB !== 'string') { + return -1; + } + if (typeofA === 'string' && typeofB === 'string') { + return a.localeCompare(b); + } + + const tokensA = a.split('.'); + const prefixA = tokensA[0]; + const tokensB = b.split('.'); + const prefixB = tokensB[0]; + + if (prefixA === resultsField && tokensA.length > 1 && prefixB !== resultsField) { + tokensA.shift(); + tokensA.shift(); + if (tokensA.join('.') === b) return 1; + return tokensA.join('.').localeCompare(b); + } + + if (prefixB === resultsField && tokensB.length > 1 && prefixA !== resultsField) { + tokensB.shift(); + tokensB.shift(); + if (tokensB.join('.') === a) return -1; + return a.localeCompare(tokensB.join('.')); + } + + return a.localeCompare(b); +}; + export function getFlattenedFields(obj: EsDocSource, resultsField: string): EsFieldName[] { const flatDocFields: EsFieldName[] = []; const newDocFields = Object.keys(obj); @@ -84,6 +184,46 @@ export function getFlattenedFields(obj: EsDocSource, resultsField: string): EsFi return flatDocFields.filter(f => f !== ML__ID_COPY); } +export const getDefaultRegressionFields = ( + docs: EsDoc[], + jobConfig: DataFrameAnalyticsConfig +): EsFieldName[] => { + const resultsField = jobConfig.dest.results_field; + if (docs.length === 0) { + return []; + } + + const newDocFields = getFlattenedFields(docs[0]._source, resultsField); + return newDocFields + .filter(k => { + if (k === `${resultsField}.is_training`) { + return true; + } + // predicted value of dependent variable + if (k === getPredictedFieldName(resultsField, jobConfig.analysis)) { + return true; + } + // actual value of dependent variable + if (k === getDependentVar(jobConfig.analysis)) { + return true; + } + if (k.split('.')[0] === resultsField) { + return false; + } + + let value = false; + docs.forEach(row => { + const source = row._source; + if (source[k] !== null) { + value = true; + } + }); + return value; + }) + .sort((a, b) => sortRegressionResultsFields(a, b, jobConfig)) + .slice(0, DEFAULT_REGRESSION_COLUMNS); +}; + export const getDefaultSelectableFields = (docs: EsDoc[], resultsField: string): EsFieldName[] => { if (docs.length === 0) { return []; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/index.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/index.ts index 774db35f2a52..112f828f9897 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/index.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/common/index.ts @@ -21,12 +21,18 @@ export { getValuesFromResponse, loadEvalData, Eval, + getPredictedFieldName, + INDEX_STATUS, + SEARCH_SIZE, } from './analytics'; export { getDefaultSelectableFields, + getDefaultRegressionFields, getFlattenedFields, sortColumns, + sortRegressionResultsColumns, + sortRegressionResultsFields, toggleSelectedField, EsId, EsDoc, diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.test.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.test.tsx index ce39cecb5b5e..92f438459128 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.test.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.test.tsx @@ -6,6 +6,7 @@ import { shallow } from 'enzyme'; import React from 'react'; +import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; jest.mock('../../../../../contexts/ui/use_ui_chrome_context'); jest.mock('ui/new_platform'); @@ -20,7 +21,9 @@ jest.mock('react', () => { describe('Data Frame Analytics: ', () => { test('Minimal initialization', () => { - const wrapper = shallow(); + const wrapper = shallow( + + ); // Without the jobConfig being loaded, the component will just return empty. expect(wrapper.text()).toMatch(''); // TODO Once React 16.9 is available we can write tests covering asynchronous hooks. diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx index 95ec33466754..11bb62dec162 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/exploration.tsx @@ -50,10 +50,13 @@ import { EsFieldName, EsDoc, MAX_COLUMNS, + INDEX_STATUS, } from '../../../../common'; import { getOutlierScoreFieldName } from './common'; -import { INDEX_STATUS, useExploreData } from './use_explore_data'; +import { useExploreData } from './use_explore_data'; +import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; +import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns'; const customColorScaleFactory = (n: number) => (t: number) => { if (t < 1 / n) { @@ -78,7 +81,7 @@ const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => ( {i18n.translate('xpack.ml.dataframe.analytics.exploration.jobIdTitle', { - defaultMessage: 'Job ID {jobId}', + defaultMessage: 'Outlier detection job ID {jobId}', values: { jobId }, })} @@ -87,9 +90,10 @@ const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => ( interface Props { jobId: string; + jobStatus: DATA_FRAME_TASK_STATE; } -export const Exploration: FC = React.memo(({ jobId }) => { +export const Exploration: FC = React.memo(({ jobId, jobStatus }) => { const [jobConfig, setJobConfig] = useState(undefined); const [pageIndex, setPageIndex] = useState(0); @@ -378,7 +382,14 @@ export const Exploration: FC = React.memo(({ jobId }) => { if (status === INDEX_STATUS.LOADED && tableItems.length === 0) { return ( - + + + + + + {getTaskStateBadge(jobStatus)} + + = React.memo(({ jobId }) => { - + + + + + + {getTaskStateBadge(jobStatus)} + + diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/use_explore_data.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/use_explore_data.ts index ee86156c50d4..2a07bc1251a3 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/use_explore_data.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/exploration/use_explore_data.ts @@ -18,19 +18,12 @@ import { getFlattenedFields, DataFrameAnalyticsConfig, EsFieldName, + INDEX_STATUS, + SEARCH_SIZE, } from '../../../../common'; import { getOutlierScoreFieldName } from './common'; -const SEARCH_SIZE = 1000; - -export enum INDEX_STATUS { - UNUSED, - LOADING, - LOADED, - ERROR, -} - type TableItem = Record; interface LoadExploreDataArg { diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_index.scss b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_index.scss new file mode 100644 index 000000000000..bb948785d3ef --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_index.scss @@ -0,0 +1 @@ +@import 'regression_exploration'; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_regression_exploration.scss b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_regression_exploration.scss new file mode 100644 index 000000000000..2faa04be0ab6 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/_regression_exploration.scss @@ -0,0 +1,3 @@ +.mlRegressionExploration__evaluateLoadingSpinner { + display: inline-block; +} diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx index c7ea3421ac5d..b9f9c07bc3d5 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/evaluate_panel.tsx @@ -7,13 +7,22 @@ import React, { FC, Fragment, useEffect, useState } from 'react'; import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiSpacer, EuiStat, EuiTitle } from '@elastic/eui'; +import { idx } from '@kbn/elastic-idx'; import { ErrorCallout } from './error_callout'; -import { getValuesFromResponse, loadEvalData, Eval } from '../../../../common'; +import { + getValuesFromResponse, + getDependentVar, + getPredictionFieldName, + loadEvalData, + Eval, + DataFrameAnalyticsConfig, +} from '../../../../common'; +import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns'; +import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; interface Props { - jobId: string; - index: string; - dependentVariable: string; + jobConfig: DataFrameAnalyticsConfig; + jobStatus: DATA_FRAME_TASK_STATE; } const meanSquaredErrorText = i18n.translate( @@ -30,23 +39,28 @@ const rSquaredText = i18n.translate( ); const defaultEval: Eval = { meanSquaredError: '', rSquared: '', error: null }; -export const EvaluatePanel: FC = ({ jobId, index, dependentVariable }) => { +export const EvaluatePanel: FC = ({ jobConfig, jobStatus }) => { const [trainingEval, setTrainingEval] = useState(defaultEval); const [generalizationEval, setGeneralizationEval] = useState(defaultEval); const [isLoadingTraining, setIsLoadingTraining] = useState(false); const [isLoadingGeneralization, setIsLoadingGeneralization] = useState(false); + const index = idx(jobConfig, _ => _.dest.index) as string; + const dependentVariable = getDependentVar(jobConfig.analysis); + const predictionFieldName = getPredictionFieldName(jobConfig.analysis); + // default is 'ml' + const resultsField = jobConfig.dest.results_field; + const loadData = async () => { setIsLoadingGeneralization(true); setIsLoadingTraining(true); - // TODO: resultsField and predictionFieldName will need to be properly passed to this function - // once the results view is in use. + const genErrorEval = await loadEvalData({ isTraining: false, index, dependentVariable, - resultsField: 'ml', - predictionFieldName: undefined, + resultsField, + predictionFieldName, }); if (genErrorEval.success === true && genErrorEval.eval) { @@ -65,14 +79,13 @@ export const EvaluatePanel: FC = ({ jobId, index, dependentVariable }) => error: genErrorEval.error, }); } - // TODO: resultsField and predictionFieldName will need to be properly passed to this function - // once the results view is in use. + const trainingErrorEval = await loadEvalData({ isTraining: true, index, dependentVariable, - resultsField: 'ml', - predictionFieldName: undefined, + resultsField, + predictionFieldName, }); if (trainingErrorEval.success === true && trainingErrorEval.eval) { @@ -99,14 +112,21 @@ export const EvaluatePanel: FC = ({ jobId, index, dependentVariable }) => return ( - - - {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', { - defaultMessage: 'Job ID {jobId}', - values: { jobId }, - })} - - + + + + + {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', { + defaultMessage: 'Regression job ID {jobId}', + values: { jobId: jobConfig.id }, + })} + + + + + {getTaskStateBadge(jobStatus)} + + diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx index f6cb010c3f04..1f7564b2fe21 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/regression_exploration.tsx @@ -4,25 +4,61 @@ * you may not use this file except in compliance with the Elastic License. */ -import React, { FC, Fragment } from 'react'; -// import { i18n } from '@kbn/i18n'; -import { EuiSpacer } from '@elastic/eui'; - +import React, { FC, Fragment, useState, useEffect } from 'react'; +import { EuiSpacer, EuiLoadingSpinner, EuiPanel } from '@elastic/eui'; +import { ml } from '../../../../../services/ml_api_service'; +import { DataFrameAnalyticsConfig } from '../../../../common'; import { EvaluatePanel } from './evaluate_panel'; -// import { ResultsTable } from './results_table'; +import { ResultsTable } from './results_table'; +import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; + +interface GetDataFrameAnalyticsResponse { + count: number; + data_frame_analytics: DataFrameAnalyticsConfig[]; +} + +const LoadingPanel: FC = () => ( + + + +); interface Props { jobId: string; - destIndex: string; - dependentVariable: string; + jobStatus: DATA_FRAME_TASK_STATE; } -export const RegressionExploration: FC = ({ jobId, destIndex, dependentVariable }) => { +export const RegressionExploration: FC = ({ jobId, jobStatus }) => { + const [jobConfig, setJobConfig] = useState(undefined); + const [isLoadingJobConfig, setIsLoadingJobConfig] = useState(false); + + useEffect(() => { + (async function() { + setIsLoadingJobConfig(true); + const analyticsConfigs: GetDataFrameAnalyticsResponse = await ml.dataFrameAnalytics.getDataFrameAnalytics( + jobId + ); + if ( + Array.isArray(analyticsConfigs.data_frame_analytics) && + analyticsConfigs.data_frame_analytics.length > 0 + ) { + setJobConfig(analyticsConfigs.data_frame_analytics[0]); + setIsLoadingJobConfig(false); + } + })(); + }, []); + return ( - + {isLoadingJobConfig === true && jobConfig === undefined && } + {isLoadingJobConfig === false && jobConfig !== undefined && ( + + )} - {/* */} + {isLoadingJobConfig === true && jobConfig === undefined && } + {isLoadingJobConfig === false && jobConfig !== undefined && ( + + )} ); }; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx new file mode 100644 index 000000000000..fdd6782bba37 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/results_table.tsx @@ -0,0 +1,482 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { Fragment, FC, useEffect, useState } from 'react'; +import moment from 'moment-timezone'; + +import { i18n } from '@kbn/i18n'; +import { + EuiBadge, + EuiButtonIcon, + EuiCallOut, + EuiCheckbox, + EuiFlexGroup, + EuiFlexItem, + EuiPanel, + EuiPopover, + EuiPopoverTitle, + EuiProgress, + EuiSpacer, + EuiText, + EuiTitle, + EuiToolTip, + Query, +} from '@elastic/eui'; + +import { Query as QueryType } from '../../../analytics_management/components/analytics_list/common'; + +import { + ColumnType, + MlInMemoryTableBasic, + OnTableChangeArg, + SortingPropType, + SORT_DIRECTION, +} from '../../../../../components/ml_in_memory_table'; + +import { formatHumanReadableDateTimeSeconds } from '../../../../../util/date_utils'; +import { SavedSearchQuery } from '../../../../../contexts/kibana'; + +import { + sortRegressionResultsColumns, + sortRegressionResultsFields, + toggleSelectedField, + DataFrameAnalyticsConfig, + EsFieldName, + EsDoc, + MAX_COLUMNS, + getPredictedFieldName, + INDEX_STATUS, +} from '../../../../common'; +import { getTaskStateBadge } from '../../../analytics_management/components/analytics_list/columns'; +import { DATA_FRAME_TASK_STATE } from '../../../analytics_management/components/analytics_list/common'; + +import { useExploreData, defaultSearchQuery } from './use_explore_data'; + +const PAGE_SIZE_OPTIONS = [5, 10, 25, 50]; + +const ExplorationTitle: React.SFC<{ jobId: string }> = ({ jobId }) => ( + + + {i18n.translate('xpack.ml.dataframe.analytics.regressionExploration.jobIdTitle', { + defaultMessage: 'Regression job ID {jobId}', + values: { jobId }, + })} + + +); + +interface Props { + jobConfig: DataFrameAnalyticsConfig; + jobStatus: DATA_FRAME_TASK_STATE; +} + +export const ResultsTable: FC = React.memo(({ jobConfig, jobStatus }) => { + const [pageIndex, setPageIndex] = useState(0); + const [pageSize, setPageSize] = useState(25); + const [clearTable, setClearTable] = useState(false); + const [selectedFields, setSelectedFields] = useState([] as EsFieldName[]); + const [isColumnsPopoverVisible, setColumnsPopoverVisible] = useState(false); + const [searchQuery, setSearchQuery] = useState(defaultSearchQuery); + const [searchError, setSearchError] = useState(undefined); + const [searchString, setSearchString] = useState(undefined); + + // EuiInMemoryTable has an issue with dynamic sortable columns + // and will trigger a full page Kibana error in such a case. + // The following is a workaround until this is solved upstream: + // - If the sortable/columns config changes, + // the table will be unmounted/not rendered. + // This is what setClearTable(true) in toggleColumn() does. + // - After that on next render it gets re-enabled. To make sure React + // doesn't consolidate the state updates, setTimeout is used. + if (clearTable) { + setTimeout(() => setClearTable(false), 0); + } + + function toggleColumnsPopover() { + setColumnsPopoverVisible(!isColumnsPopoverVisible); + } + + function closeColumnsPopover() { + setColumnsPopoverVisible(false); + } + + function toggleColumn(column: EsFieldName) { + if (tableItems.length > 0 && jobConfig !== undefined) { + setClearTable(true); + // spread to a new array otherwise the component wouldn't re-render + setSelectedFields([...toggleSelectedField(selectedFields, column)]); + } + } + + const { + errorMessage, + loadExploreData, + sortField, + sortDirection, + status, + tableItems, + } = useExploreData(jobConfig, selectedFields, setSelectedFields); + + let docFields: EsFieldName[] = []; + let docFieldsCount = 0; + if (tableItems.length > 0) { + docFields = Object.keys(tableItems[0]); + docFields.sort((a, b) => sortRegressionResultsFields(a, b, jobConfig)); + docFieldsCount = docFields.length; + } + + const columns: ColumnType[] = []; + + if (jobConfig !== undefined && selectedFields.length > 0 && tableItems.length > 0) { + columns.push( + ...selectedFields.sort(sortRegressionResultsColumns(tableItems[0], jobConfig)).map(k => { + const column: ColumnType = { + field: k, + name: k, + sortable: true, + truncateText: true, + }; + + const render = (d: any, fullItem: EsDoc) => { + if (Array.isArray(d) && d.every(item => typeof item === 'string')) { + // If the cells data is an array of strings, return as a comma separated list. + // The list will get limited to 5 items with `…` at the end if there's more in the original array. + return `${d.slice(0, 5).join(', ')}${d.length > 5 ? ', …' : ''}`; + } else if (Array.isArray(d)) { + // If the cells data is an array of e.g. objects, display a 'array' badge with a + // tooltip that explains that this type of field is not supported in this table. + return ( + + + {i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.indexArrayBadgeContent', + { + defaultMessage: 'array', + } + )} + + + ); + } else if (typeof d === 'object' && d !== null) { + // If the cells data is an object, display a 'object' badge with a + // tooltip that explains that this type of field is not supported in this table. + return ( + + + {i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.indexObjectBadgeContent', + { + defaultMessage: 'object', + } + )} + + + ); + } + + return d; + }; + + let columnType; + + if (tableItems.length > 0) { + columnType = typeof tableItems[0][k]; + } + + if (typeof columnType !== 'undefined') { + switch (columnType) { + case 'boolean': + column.dataType = 'boolean'; + break; + case 'Date': + column.align = 'right'; + column.render = (d: any) => + formatHumanReadableDateTimeSeconds(moment(d).unix() * 1000); + break; + case 'number': + column.dataType = 'number'; + column.render = render; + break; + default: + column.render = render; + break; + } + } else { + column.render = render; + } + + return column; + }) + ); + } + + useEffect(() => { + if (jobConfig !== undefined) { + const predictedFieldName = getPredictedFieldName( + jobConfig.dest.results_field, + jobConfig.analysis + ); + const predictedFieldSelected = selectedFields.includes(predictedFieldName); + + const field = predictedFieldSelected ? predictedFieldName : selectedFields[0]; + const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC; + loadExploreData({ field, direction, searchQuery }); + return; + } + }, [JSON.stringify(searchQuery)]); + + useEffect(() => { + // by default set the sorting to descending on the prediction field (`_prediction`). + // if that's not available sort ascending on the first column. + // also check if the current sorting field is still available. + if (jobConfig !== undefined && columns.length > 0 && !selectedFields.includes(sortField)) { + const predictedFieldName = getPredictedFieldName( + jobConfig.dest.results_field, + jobConfig.analysis + ); + const predictedFieldSelected = selectedFields.includes(predictedFieldName); + + const field = predictedFieldSelected ? predictedFieldName : selectedFields[0]; + const direction = predictedFieldSelected ? SORT_DIRECTION.DESC : SORT_DIRECTION.ASC; + loadExploreData({ field, direction, searchQuery }); + return; + } + }, [jobConfig, columns.length, sortField, sortDirection, tableItems.length]); + + let sorting: SortingPropType = false; + let onTableChange; + + if (columns.length > 0 && sortField !== '' && sortField !== undefined) { + sorting = { + sort: { + field: sortField, + direction: sortDirection, + }, + }; + + onTableChange = ({ + page = { index: 0, size: 10 }, + sort = { field: sortField, direction: sortDirection }, + }: OnTableChangeArg) => { + const { index, size } = page; + setPageIndex(index); + setPageSize(size); + + if (sort.field !== sortField || sort.direction !== sortDirection) { + setClearTable(true); + loadExploreData({ ...sort, searchQuery }); + } + }; + } + + const pagination = { + initialPageIndex: pageIndex, + initialPageSize: pageSize, + totalItemCount: tableItems.length, + pageSizeOptions: PAGE_SIZE_OPTIONS, + hidePerPageOptions: false, + }; + + const onQueryChange = ({ query, error }: { query: QueryType; error: any }) => { + if (error) { + setSearchError(error.message); + } else { + try { + const esQueryDsl = Query.toESQuery(query); + setSearchQuery(esQueryDsl); + setSearchString(query.text); + setSearchError(undefined); + } catch (e) { + setSearchError(e.toString()); + } + } + }; + + const search = { + onChange: onQueryChange, + defaultQuery: searchString, + box: { + incremental: false, + placeholder: i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.searchBoxPlaceholder', + { + defaultMessage: 'E.g. avg>0.5', + } + ), + }, + filters: [ + { + type: 'field_value_toggle_group', + field: `${jobConfig.dest.results_field}.is_training`, + items: [ + { + value: false, + name: i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.isTestingLabel', + { + defaultMessage: 'Testing', + } + ), + }, + { + value: true, + name: i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.isTrainingLabel', + { + defaultMessage: 'Training', + } + ), + }, + ], + }, + ], + }; + + if (jobConfig === undefined) { + return null; + } + + if (status === INDEX_STATUS.ERROR) { + return ( + + + + + + + {getTaskStateBadge(jobStatus)} + + + +

{errorMessage}

+
+
+ ); + } + + return ( + + + + + + + + + {getTaskStateBadge(jobStatus)} + + + + + + + {docFieldsCount > MAX_COLUMNS && ( + + {i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.fieldSelection', + { + defaultMessage: + '{selectedFieldsLength, number} of {docFieldsCount, number} {docFieldsCount, plural, one {field} other {fields}} selected', + values: { selectedFieldsLength: selectedFields.length, docFieldsCount }, + } + )} + + )} + + + + + } + isOpen={isColumnsPopoverVisible} + closePopover={closeColumnsPopover} + ownFocus + > + + {i18n.translate( + 'xpack.ml.dataframe.analytics.regressionExploration.selectFieldsPopoverTitle', + { + defaultMessage: 'Select fields', + } + )} + +
+ {docFields.map(d => ( + toggleColumn(d)} + disabled={selectedFields.includes(d) && selectedFields.length === 1} + /> + ))} +
+
+
+
+
+
+
+ {status === INDEX_STATUS.LOADING && } + {status !== INDEX_STATUS.LOADING && ( + + )} + {clearTable === false && (columns.length > 0 || searchQuery !== defaultSearchQuery) && ( + + + + + )} +
+ ); +}); diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts new file mode 100644 index 000000000000..3e7266eb8947 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/components/regression_exploration/use_explore_data.ts @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useEffect, useState } from 'react'; + +import { SearchResponse } from 'elasticsearch'; + +import { SortDirection, SORT_DIRECTION } from '../../../../../components/ml_in_memory_table'; + +import { ml } from '../../../../../services/ml_api_service'; +import { getNestedProperty } from '../../../../../util/object_utils'; +import { SavedSearchQuery } from '../../../../../contexts/kibana'; + +import { + getDefaultRegressionFields, + getFlattenedFields, + DataFrameAnalyticsConfig, + EsFieldName, + getPredictedFieldName, + INDEX_STATUS, + SEARCH_SIZE, +} from '../../../../common'; + +export const defaultSearchQuery = { + match_all: {}, +}; + +type TableItem = Record; + +interface LoadExploreDataArg { + field: string; + direction: SortDirection; + searchQuery: SavedSearchQuery; +} +export interface UseExploreDataReturnType { + errorMessage: string; + loadExploreData: (arg: LoadExploreDataArg) => void; + sortField: EsFieldName; + sortDirection: SortDirection; + status: INDEX_STATUS; + tableItems: TableItem[]; +} + +interface SearchQuery { + query: SavedSearchQuery; + sort?: any; +} + +export const useExploreData = ( + jobConfig: DataFrameAnalyticsConfig | undefined, + selectedFields: EsFieldName[], + setSelectedFields: React.Dispatch> +): UseExploreDataReturnType => { + const [errorMessage, setErrorMessage] = useState(''); + const [status, setStatus] = useState(INDEX_STATUS.UNUSED); + const [tableItems, setTableItems] = useState([]); + const [sortField, setSortField] = useState(''); + const [sortDirection, setSortDirection] = useState(SORT_DIRECTION.ASC); + + const loadExploreData = async ({ field, direction, searchQuery }: LoadExploreDataArg) => { + if (jobConfig !== undefined) { + setErrorMessage(''); + setStatus(INDEX_STATUS.LOADING); + + try { + const resultsField = jobConfig.dest.results_field; + const body: SearchQuery = { + query: searchQuery, + }; + + if (field !== undefined) { + body.sort = [ + { + [field]: { + order: direction, + }, + }, + ]; + } + + const resp: SearchResponse = await ml.esSearch({ + index: jobConfig.dest.index, + size: SEARCH_SIZE, + body, + }); + + setSortField(field); + setSortDirection(direction); + + const docs = resp.hits.hits; + + if (docs.length === 0) { + setTableItems([]); + setStatus(INDEX_STATUS.LOADED); + return; + } + + if (selectedFields.length === 0) { + const newSelectedFields = getDefaultRegressionFields(docs, jobConfig); + setSelectedFields(newSelectedFields); + } + + // Create a version of the doc's source with flattened field names. + // This avoids confusion later on if a field name has dots in its name + // or is a nested fields when displaying it via EuiInMemoryTable. + const flattenedFields = getFlattenedFields(docs[0]._source, resultsField); + const transformedTableItems = docs.map(doc => { + const item: TableItem = {}; + flattenedFields.forEach(ff => { + item[ff] = getNestedProperty(doc._source, ff); + if (item[ff] === undefined) { + // If the attribute is undefined, it means it was not a nested property + // but had dots in its actual name. This selects the property by its + // full name and assigns it to `item[ff]`. + item[ff] = doc._source[`"${ff}"`]; + } + if (item[ff] === undefined) { + const parts = ff.split('.'); + if (parts[0] === resultsField && parts.length >= 2) { + parts.shift(); + if (doc._source[resultsField] !== undefined) { + item[ff] = doc._source[resultsField][parts.join('.')]; + } + } + } + }); + return item; + }); + + setTableItems(transformedTableItems); + setStatus(INDEX_STATUS.LOADED); + } catch (e) { + if (e.message !== undefined) { + setErrorMessage(e.message); + } else { + setErrorMessage(JSON.stringify(e)); + } + setTableItems([]); + setStatus(INDEX_STATUS.ERROR); + } + } + }; + + useEffect(() => { + if (jobConfig !== undefined) { + loadExploreData({ + field: getPredictedFieldName(jobConfig.dest.results_field, jobConfig.analysis), + direction: SORT_DIRECTION.DESC, + searchQuery: defaultSearchQuery, + }); + } + }, [jobConfig && jobConfig.id]); + + return { errorMessage, loadExploreData, sortField, sortDirection, status, tableItems }; +}; diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/directive.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/directive.tsx index ca448aaa7b93..4a28e0b0b881 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/directive.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/directive.tsx @@ -53,8 +53,7 @@ module.directive('mlDataFrameAnalyticsExploration', ($injector: InjectorService) , diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/page.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/page.tsx index 0bbc31370417..b3d13db0a355 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/page.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_exploration/page.tsx @@ -26,13 +26,13 @@ import { Exploration } from './components/exploration'; import { RegressionExploration } from './components/regression_exploration'; import { ANALYSIS_CONFIG_TYPE } from '../../common/analytics'; +import { DATA_FRAME_TASK_STATE } from '../analytics_management/components/analytics_list/common'; export const Page: FC<{ jobId: string; - analysisType: string; - destIndex: string; - depVar: string; -}> = ({ jobId, analysisType, destIndex, depVar }) => ( + analysisType: ANALYSIS_CONFIG_TYPE; + jobStatus: DATA_FRAME_TASK_STATE; +}> = ({ jobId, analysisType, jobStatus }) => ( @@ -66,9 +66,11 @@ export const Page: FC<{ - {analysisType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION && } + {analysisType === ANALYSIS_CONFIG_TYPE.OUTLIER_DETECTION && ( + + )} {analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION && ( - + )} diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/actions.tsx b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/actions.tsx index 7ad86c5d380c..5e5283f9e6c4 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/actions.tsx +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/actions.tsx @@ -13,7 +13,7 @@ import { createPermissionFailureMessage, } from '../../../../../privilege/check_privilege'; -import { isOutlierAnalysis, getAnalysisType, getDependentVar } from '../../../../common/analytics'; +import { getAnalysisType } from '../../../../common/analytics'; import { getResultsUrl, isDataFrameAnalyticsRunning, DataFrameAnalyticsListRow } from './common'; import { stopAnalytics } from '../../services/analytics_service'; @@ -25,14 +25,11 @@ export const AnalyticsViewAction = { isPrimary: true, render: (item: DataFrameAnalyticsListRow) => { const analysisType = getAnalysisType(item.config.analysis); - const destIndex = item.config.dest.index; - const dependentVariable = getDependentVar(item.config.analysis); + const jobStatus = item.stats.state; - const url = getResultsUrl(item.id, analysisType, destIndex, dependentVariable); - // Disable 'View' link for regression until results view is complete + const url = getResultsUrl(item.id, analysisType, jobStatus); return ( (window.location.href = url)} size="xs" color="text" diff --git a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts index 99d1889b265e..098c239e4f6b 100644 --- a/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts +++ b/x-pack/legacy/plugins/ml/public/data_frame_analytics/pages/analytics_management/components/analytics_list/common.ts @@ -5,7 +5,6 @@ */ import { DataFrameAnalyticsId, DataFrameAnalyticsConfig } from '../../../../common'; -import { ANALYSIS_CONFIG_TYPE } from '../../../../common/analytics'; export enum DATA_FRAME_TASK_STATE { ANALYZING = 'analyzing', @@ -118,17 +117,6 @@ export function isCompletedAnalyticsJob(stats: DataFrameAnalyticsStats) { return stats.state === DATA_FRAME_TASK_STATE.STOPPED && progress === 100; } -export function getResultsUrl( - jobId: string, - analysisType: string, - destIndex: string = '', - dependentVariable: string = '' -) { - const destIndexParam = `,destIndex:${destIndex}`; - const depVarParam = `,depVar:${dependentVariable}`; - const isRegression = analysisType === ANALYSIS_CONFIG_TYPE.REGRESSION; - - return `ml#/data_frame_analytics/exploration?_g=(ml:(jobId:${jobId},analysisType:${analysisType}${ - isRegression && destIndex !== '' ? destIndexParam : '' - }${isRegression && dependentVariable !== '' ? depVarParam : ''}))`; +export function getResultsUrl(jobId: string, analysisType: string, status: DATA_FRAME_TASK_STATE) { + return `ml#/data_frame_analytics/exploration?_g=(ml:(jobId:${jobId},analysisType:${analysisType},jobStatus:${status}))`; }