From 4260d82c97618443b502557e3c4f92051b244515 Mon Sep 17 00:00:00 2001 From: Melissa Alvarez Date: Thu, 13 Jun 2019 09:36:51 -0400 Subject: [PATCH] [ML] File datavisualizer - custom timestamp override functionality (#38727) * show custom input option for timestamp format * show validation errors if custom timestamp format invalid * Update error messages. Remove unnecessary comments * use custom constant for delimiter override * fix i18n errors --- .../__snapshots__/overrides.test.js.snap | 84 +----------- .../edit_flyout/options/option_lists.js | 35 +---- .../components/edit_flyout/overrides.js | 66 +++++++++- .../edit_flyout/overrides_validation.js | 124 ++++++++++++++++++ 4 files changed, 194 insertions(+), 115 deletions(-) create mode 100644 x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides_validation.js diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap index 37d0e0a6cba8..d298cf111287 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap @@ -90,6 +90,9 @@ exports[`Overrides render overrides 1`] = ` onChange={[Function]} options={ Array [ + Object { + "label": "custom", + }, Object { "label": "dd/MMM/yyyy:HH:mm:ss XX", }, @@ -279,87 +282,6 @@ exports[`Overrides render overrides 1`] = ` Object { "label": "yyyy-MM-dd HH:mm:ss:SSSSSSSSSXXX", }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSS", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSSXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSSXXX", - }, - Object { - "label": "yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSSXXX", - }, Object { "label": "yyyy-MM-dd HH:mm:ssXX", }, diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/options/option_lists.js b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/options/option_lists.js index 92dd7bcefb70..d8daab46f355 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/options/option_lists.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/options/option_lists.js @@ -12,7 +12,10 @@ export const FORMAT_OPTIONS = [ // 'xml', ]; +export const CUSTOM_DROPDOWN_OPTION = 'custom'; + export const TIMESTAMP_OPTIONS = [ + CUSTOM_DROPDOWN_OPTION, 'dd/MMM/yyyy:HH:mm:ss XX', 'EEE MMM dd HH:mm zzz yyyy', 'EEE MMM dd HH:mm:ss yyyy', @@ -96,36 +99,6 @@ export const TIMESTAMP_OPTIONS = [ 'yyyy-MM-dd HH:mm:ss:SSSSSSXXX', 'yyyy-MM-dd HH:mm:ss:SSSSSSSSSXXX', - `yyyy-MM-dd'T'HH:mm:ss,SSS`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSS`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSS`, - `yyyy-MM-dd'T'HH:mm:ss.SSS`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSS`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS`, - `yyyy-MM-dd'T'HH:mm:ss:SSS`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSS`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSS`, - - `yyyy-MM-dd'T'HH:mm:ss,SSSXX`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSSXX`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSSXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSSXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSSXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSSXX`, - - `yyyy-MM-dd'T'HH:mm:ss,SSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss,SSSSSSSSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSSXXX`, - `yyyy-MM-dd'T'HH:mm:ss:SSSSSSSSSXXX`, - 'yyyy-MM-dd HH:mm:ssXX', 'yyyy-MM-dd HH:mm:ssXXX', 'yyyyMMddHHmmss', @@ -137,7 +110,7 @@ export const DELIMITER_OPTIONS = [ 'semicolon', 'pipe', 'space', - 'other', + CUSTOM_DROPDOWN_OPTION, ]; export const QUOTE_OPTIONS = [ diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js index 9129032df826..7a619df46c76 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js @@ -30,6 +30,9 @@ import { getQuoteOptions, // getCharsetOptions, } from './options'; +import { isTimestampFormatValid } from './overrides_validation'; + +import { TIMESTAMP_OPTIONS, CUSTOM_DROPDOWN_OPTION } from './options/option_lists'; const formatOptions = getFormatOptions(); const timestampFormatOptions = getTimestampFormatOptions(); @@ -55,6 +58,11 @@ export class Overrides extends Component { } }); + customTimestampFormatErrors = i18n.translate('xpack.ml.fileDatavisualizer.editFlyout.overrides.customTimestampFormatErrorMessage', { + defaultMessage: `Timestamp format must be a combination of these Java date/time formats: + yy, yyyy, M, MM, MMM, MMMM, d, dd, EEE, EEEE, H, HH, h, mm, ss, S through SSSSSSSSS, a, XX, XXX, zzz` + }); + static getDerivedStateFromProps(props, state) { const { originalSettings } = props; @@ -99,16 +107,31 @@ export class Overrides extends Component { return { originalColumnNames, customDelimiter: (customD === undefined) ? '' : customD, + customTimestampFormat: '', linesToSampleValid: true, + timestampFormatValid: true, + timestampFormatError: null, overrides, ...state, }; } componentDidMount() { + const originalTimestampFormat = (this.props && this.props.originalSettings && this.props.originalSettings.timestampFormat); + if (typeof this.props.setApplyOverrides === 'function') { this.props.setApplyOverrides(this.applyOverrides); } + + if (originalTimestampFormat !== undefined) { + const optionExists = (TIMESTAMP_OPTIONS.some(option => option === originalTimestampFormat)); + if (optionExists === false) { + // Incoming format does not exist in dropdown. Display custom input with incoming format as default value. + const overrides = { ...this.state.overrides }; + overrides.timestampFormat = CUSTOM_DROPDOWN_OPTION; + this.setState({ customTimestampFormat: originalTimestampFormat, overrides }); + } + } } componentWillUnmount() { @@ -120,6 +143,9 @@ export class Overrides extends Component { applyOverrides = () => { const overrides = { ...this.state.overrides }; overrides.delimiter = convertDelimiterBack(overrides.delimiter, this.state.customDelimiter); + if (overrides.timestampFormat === CUSTOM_DROPDOWN_OPTION && this.state.customTimestampFormat !== '') { + overrides.timestampFormat = this.state.customTimestampFormat; + } this.props.setOverrides(overrides); } @@ -137,6 +163,17 @@ export class Overrides extends Component { onTimestampFormatChange = ([opt]) => { const timestampFormat = opt ? opt.label : ''; this.setOverride({ timestampFormat }); + if (opt !== CUSTOM_DROPDOWN_OPTION) { + this.props.setOverridesValid(true); + } + } + + onCustomTimestampFormatChange = (e) => { + this.setState({ customTimestampFormat: e.target.value }); + // check whether the value is valid and set that to state. + const { isValid, errorMessage } = isTimestampFormatValid(e.target.value); + this.setState({ timestampFormatValid: isValid, timestampFormatError: errorMessage }); + this.props.setOverridesValid(isValid); } onTimestampFieldChange = ([opt]) => { @@ -199,8 +236,11 @@ export class Overrides extends Component { const { fields } = this.props; const { customDelimiter, + customTimestampFormat, originalColumnNames, linesToSampleValid, + timestampFormatError, + timestampFormatValid, overrides, } = this.state; @@ -219,6 +259,7 @@ export class Overrides extends Component { } = overrides; const fieldOptions = getSortedFields(fields); + const timestampFormatErrorsList = [this.customTimestampFormatErrors, timestampFormatError]; return ( @@ -276,7 +317,7 @@ export class Overrides extends Component { /> { - (delimiter === 'other') && + (delimiter === CUSTOM_DROPDOWN_OPTION) && + { + (timestampFormat === CUSTOM_DROPDOWN_OPTION) && + + } + > + + + } = 0) { + result.isValid = false; + result.errorMessage = i18n.translate('xpack.ml.fileDatavisualizer.editFlyout.overrides.timestampQuestionMarkValidationErrorMessage', { + defaultMessage: 'Timestamp format {timestampFormat} not supported because it contains a question mark character ({fieldPlaceholder})', + values: { + timestampFormat, + fieldPlaceholder: '?', + } + }); + return result; + } + + let notQuoted = true; + let prevChar = null; + let prevLetterGroup = null; + let pos = 0; + + while (pos < timestampFormat.length) { + const curChar = timestampFormat.charAt(pos); + + if (curChar === '\'') { + notQuoted = !notQuoted; + } else if (notQuoted && isLetter(curChar)) { + const startPos = pos; + let endPos = startPos + 1; + while (endPos < timestampFormat.length && timestampFormat.charAt(endPos) === curChar) { + ++endPos; + ++pos; + } + + const letterGroup = timestampFormat.substring(startPos, endPos); + + if (VALID_LETTER_GROUPS[letterGroup] !== true) { + const length = letterGroup.length; + // Special case of fractional seconds + if (curChar !== 'S' || FRACTIONAL_SECOND_SEPARATORS.indexOf(prevChar) === -1 || + !('ss' === prevLetterGroup) || endPos - startPos > 9) { + result.isValid = false; + + result.errorMessage = i18n.translate('xpack.ml.fileDatavisualizer.editFlyout.overrides.timestampLetterValidationErrorMessage', { + defaultMessage: 'Letter { length, plural, one { {lg} } other { group {lg} } } in {format} is not supported', + values: { + length, + lg: letterGroup, + format: timestampFormat + }, + }); + + if (curChar === 'S') { + // disable exceeds maximum line length error so i18n check passes + result.errorMessage = i18n.translate( + 'xpack.ml.fileDatavisualizer.editFlyout.overrides.timestampLetterSValidationErrorMessage', + { + defaultMessage: 'Letter { length, plural, one { {lg} } other { group {lg} } } in {format} is not supported because it is not preceded by ss and a separator from {sep}', // eslint-disable-line + values: { + length, + lg: letterGroup, + sep: FRACTIONAL_SECOND_SEPARATORS, + format: timestampFormat + }, + }); + } + + return result; + } + } + prevLetterGroup = letterGroup; + } + + prevChar = curChar; + ++pos; + } + + if (prevLetterGroup == null) { + result.isValid = false; + result.errorMessage = i18n.translate('xpack.ml.fileDatavisualizer.editFlyout.overrides.timestampEmptyValidationErrorMessage', { + defaultMessage: 'No time format letter groups in timestamp format {timestampFormat}', + values: { + timestampFormat, + } + }); + } + + return result; +}