[ML] Adding ability to override number of sample lines in File Data Visualizer (#29214)

* [ML] Adding ability to override number of sample lines in file data viz

* tiny tweak

* updating tests

x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/__snapshots__/overrides.test.js.snap

@@ -2,6 +2,28 @@
 
 exports[`Overrides render overrides 1`] = `
 <EuiForm>
+  <EuiFormRow
+    describedByIds={Array []}
+    error="Value must be greater than 3 and less than or equal to 1000000"
+    fullWidth={false}
+    hasEmptyLabelSpace={false}
+    isInvalid={false}
+    label={
+      <FormattedMessage
+        defaultMessage="Number of lines to sample"
+        id="xpack.ml.fileDatavisualizer.editFlyout.overrides.linesToSampleFormRowLabel"
+        values={Object {}}
+      />
+    }
+  >
+    <EuiFieldNumber
+      compressed={false}
+      fullWidth={false}
+      isInvalid={false}
+      isLoading={false}
+      onChange={[Function]}
+    />
+  </EuiFormRow>
   <EuiFormRow
     describedByIds={Array []}
     fullWidth={false}

x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/edit_flyout.js

@@ -29,6 +29,9 @@ export class EditFlyout extends Component {
     super(props);
 
     this.applyOverrides = () => {};
+    this.state = {
+      overridesValid: true
+    };
   }
 
   applyAndClose = () => {
@@ -42,10 +45,14 @@ export class EditFlyout extends Component {
   unsetApplyOverrides = () => {
     this.applyOverrides = () => {};
   }
+  setOverridesValid = (overridesValid) => {
+    this.setState({ overridesValid });
+  }
 
   render() {
-    const { isFlyoutVisible, closeEditFlyout } = this.props;
     const {
+      isFlyoutVisible,
+      closeEditFlyout,
       setOverrides,
       overrides,
       originalSettings,
@@ -78,6 +85,7 @@ export class EditFlyout extends Component {
                 overrides={overrides}
                 originalSettings={originalSettings}
                 setApplyOverrides={this.setApplyOverrides}
+                setOverridesValid={this.setOverridesValid}
                 fields={fields}
               />
 
@@ -105,8 +113,8 @@ export class EditFlyout extends Component {
                 <EuiFlexItem grow={false}>
                   <EuiButton
                     onClick={this.applyAndClose}
+                    isDisabled={(this.state.overridesValid === false)}
                     fill
-                    // isDisabled={(isValidJobDetails === false) || (isValidJobCustomUrls === false)}
                   >
                     <FormattedMessage
                       id="xpack.ml.fileDatavisualizer.editFlyout.applyOverrideSettingsButtonLabel"

x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.js

@@ -6,6 +6,7 @@
 
 
 import { FormattedMessage } from '@kbn/i18n/react';
+import { i18n } from '@kbn/i18n';
 import React, {
   Component,
 } from 'react';
@@ -19,6 +20,7 @@ import {
   EuiSpacer,
   EuiTitle,
   EuiTextArea,
+  EuiFieldNumber,
 } from '@elastic/eui';
 
 import {
@@ -35,6 +37,9 @@ const delimiterOptions = getDelimiterOptions();
 const quoteOptions = getQuoteOptions();
 // const charsetOptions = getCharsetOptions();
 
+const LINES_TO_SAMPLE_VALUE_MIN = 3;
+const LINES_TO_SAMPLE_VALUE_MAX = 1000000;
+
 export class Overrides extends Component {
   constructor(props) {
     super(props);
@@ -42,6 +47,14 @@ export class Overrides extends Component {
     this.state = {};
   }
 
+  linesToSampleErrors = i18n.translate('xpack.ml.fileDatavisualizer.editFlyout.overrides.linesToSampleErrorMessage', {
+    defaultMessage: 'Value must be greater than {min} and less than or equal to {max}',
+    values: {
+      min: LINES_TO_SAMPLE_VALUE_MIN,
+      max: LINES_TO_SAMPLE_VALUE_MAX,
+    }
+  });
+
   static getDerivedStateFromProps(props, state) {
     const { originalSettings } = props;
 
@@ -56,6 +69,7 @@ export class Overrides extends Component {
       grokPattern,
       timestampField,
       timestampFormat,
+      linesToSample,
     } = props.overrides;
 
     const {
@@ -68,22 +82,27 @@ export class Overrides extends Component {
       originalColumnNames
     } = getColumnNames(columnNames, originalSettings);
 
-    const initialState =  {
+    const overrides =  {
       charset: (charset === undefined) ? originalSettings.charset : charset,
       format: (format === undefined) ? originalSettings.format : format,
       hasHeaderRow: (hasHeaderRow === undefined) ? originalSettings.hasHeaderRow : hasHeaderRow,
       columnNames: newColumnNames,
-      originalColumnNames,
       delimiter: d,
-      customDelimiter: (customD === undefined) ? '' : customD,
       quote: (quote === undefined) ? originalSettings.quote : quote,
       shouldTrimFields: (shouldTrimFields === undefined) ? originalSettings.shouldTrimFields : shouldTrimFields,
       grokPattern: (grokPattern === undefined) ? originalSettings.grokPattern : grokPattern,
       timestampFormat: (timestampFormat === undefined) ? originalSettings.timestampFormat : timestampFormat,
       timestampField: (timestampField === undefined) ? originalSettings.timestampField : timestampField,
+      linesToSample: (linesToSample === undefined) ? originalSettings.linesToSample : +linesToSample,
     };
 
-    return { ...initialState, ...state };
+    return {
+      originalColumnNames,
+      customDelimiter: (customD === undefined) ? '' : customD,
+      linesToSampleValid: true,
+      overrides,
+      ...state,
+    };
   }
 
   componentDidMount() {
@@ -99,28 +118,31 @@ export class Overrides extends Component {
   }
 
   applyOverrides = () => {
-    const overrides = { ...this.state };
-    overrides.delimiter = convertDelimiterBack(overrides);
-    delete overrides.customDelimiter;
-    delete overrides.originalColumnNames;
+    const overrides = { ...this.state.overrides };
+    overrides.delimiter = convertDelimiterBack(overrides.delimiter, this.state.customDelimiter);
 
     this.props.setOverrides(overrides);
   }
 
+  setOverride(o) {
+    const overrides = { ...this.state.overrides, ...o };
+    this.setState({ overrides });
+  }
+
   onFormatChange = (format) => {
-    this.setState({ format });
+    this.setOverride({ format });
   }
 
   onTimestampFormatChange = (timestampFormat) => {
-    this.setState({ timestampFormat });
+    this.setOverride({ timestampFormat });
   }
 
   onTimestampFieldChange = (timestampField) => {
-    this.setState({ timestampField });
+    this.setOverride({ timestampField });
   }
 
   onDelimiterChange = (delimiter) => {
-    this.setState({ delimiter });
+    this.setOverride({ delimiter });
   }
 
   onCustomDelimiterChange = (e) => {
@@ -128,54 +150,90 @@ export class Overrides extends Component {
   }
 
   onQuoteChange = (quote) => {
-    this.setState({ quote });
+    this.setOverride({ quote });
   }
 
   onHasHeaderRowChange = (e) => {
-    this.setState({ hasHeaderRow: e.target.checked });
+    this.setOverride({ hasHeaderRow: e.target.checked });
   }
 
   onShouldTrimFieldsChange = (e) => {
-    this.setState({ shouldTrimFields: e.target.checked });
+    this.setOverride({ shouldTrimFields: e.target.checked });
   }
 
   onCharsetChange = (charset) => {
-    this.setState({ charset });
+    this.setOverride({ charset });
   }
 
   onColumnNameChange = (e, i) => {
-    const columnNames = this.state.columnNames;
+    const columnNames = this.state.overrides.columnNames;
     columnNames[i] = e.target.value;
-    this.setState({ columnNames });
+    this.setOverride({ columnNames });
   }
 
   grokPatternChange = (e) => {
-    this.setState({ grokPattern: e.target.value });
+    this.setOverride({ grokPattern: e.target.value });
+  }
+
+  onLinesToSampleChange = (e) => {
+    const linesToSample = +e.target.value;
+    this.setOverride({ linesToSample });
+
+    // check whether the value is valid and set that to state.
+    const linesToSampleValid = isLinesToSampleValid(linesToSample);
+    this.setState({ linesToSampleValid });
+
+    // set the overrides valid setting in the parent component,
+    // used to disable the Apply button if any of the overrides are invalid
+    this.props.setOverridesValid(linesToSampleValid);
   }
 
 
   render() {
     const { fields } = this.props;
+    const {
+      customDelimiter,
+      originalColumnNames,
+      linesToSampleValid,
+      overrides,
+    } = this.state;
+
     const {
       timestampFormat,
       timestampField,
       format,
       delimiter,
-      customDelimiter,
       quote,
       hasHeaderRow,
       shouldTrimFields,
       // charset,
       columnNames,
-      originalColumnNames,
       grokPattern,
-    } = this.state;
+      linesToSample,
+    } = overrides;
 
     const fieldOptions = fields.map(f => ({ value: f, inputDisplay: f }));
 
     return (
 
       <EuiForm>
+        <EuiFormRow
+          error={this.linesToSampleErrors}
+          isInvalid={(linesToSampleValid === false)}
+          label={
+            <FormattedMessage
+              id="xpack.ml.fileDatavisualizer.editFlyout.overrides.linesToSampleFormRowLabel"
+              defaultMessage="Number of lines to sample"
+            />
+          }
+        >
+          <EuiFieldNumber
+            value={linesToSample}
+            onChange={this.onLinesToSampleChange}
+            isInvalid={(linesToSampleValid === false)}
+          />
+        </EuiFormRow>
+
         <EuiFormRow
           label={
             <FormattedMessage
@@ -191,7 +249,7 @@ export class Overrides extends Component {
           />
         </EuiFormRow>
         {
-          (this.state.format === 'delimited') &&
+          (format === 'delimited') &&
           <React.Fragment>
             <EuiFormRow
               label={
@@ -271,7 +329,7 @@ export class Overrides extends Component {
           </React.Fragment>
         }
         {
-          (this.state.format === 'semi_structured_text') &&
+          (format === 'semi_structured_text') &&
           <React.Fragment>
             <EuiFormRow
               label={
@@ -329,7 +387,7 @@ export class Overrides extends Component {
           />
         </EuiFormRow> */}
         {
-          (this.state.format === 'delimited' && originalColumnNames.length > 0) &&
+          (format === 'delimited' && originalColumnNames.length > 0) &&
 
           <React.Fragment>
             <EuiSpacer />
@@ -398,7 +456,7 @@ function convertDelimiter(d) {
 }
 
 // Convert the delimiter textual descriptions back to their real characters.
-function convertDelimiterBack({ delimiter, customDelimiter }) {
+function convertDelimiterBack(delimiter, customDelimiter) {
   switch (delimiter) {
     case 'comma':
       return ',';
@@ -429,3 +487,7 @@ function getColumnNames(columnNames, originalSettings) {
     originalColumnNames,
   };
 }
+
+function isLinesToSampleValid(linesToSample) {
+  return (linesToSample > LINES_TO_SAMPLE_VALUE_MIN && linesToSample <= LINES_TO_SAMPLE_VALUE_MAX);
+}

x-pack/plugins/ml/public/file_datavisualizer/components/edit_flyout/overrides.test.js

@@ -44,11 +44,11 @@ describe('Overrides', () => {
       <Overrides {...props} />
     );
 
-    expect(component.state('format')).toEqual(FORMAT_1);
+    expect(component.state('overrides').format).toEqual(FORMAT_1);
 
     component.instance().onFormatChange(FORMAT_2);
 
-    expect(component.state('format')).toEqual(FORMAT_2);
+    expect(component.state('overrides').format).toEqual(FORMAT_2);
 
   });
 });

x-pack/plugins/ml/public/file_datavisualizer/components/file_datavisualizer_view/file_datavisualizer_view.js

@@ -150,6 +150,7 @@ export class FileDataVisualizerView extends Component {
       }
 
       if (serverOverrides === undefined) {
+        // if no overrides were used, store all the settings returned from the endpoint
         this.originalSettings = serverSettings;
       } else {
         Object.keys(serverOverrides).forEach((o) => {
@@ -164,7 +165,7 @@ export class FileDataVisualizerView extends Component {
         Object.keys(serverSettings).forEach((o) => {
           const value = serverSettings[o];
           if (
-            this.overrides[o] === undefined &&
+            (this.overrides[o] === undefined) &&
             (Array.isArray(value) && (isEqual(value, this.originalSettings[o]) === false) ||
             (value !== this.originalSettings[o]))
           ) {

x-pack/plugins/ml/public/file_datavisualizer/components/utils/overrides.js

@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+export const DEFAULT_LINES_TO_SAMPLE = 1000;
 
 export const overrideDefaults = {
   timestampFormat: undefined,
@@ -16,4 +17,5 @@ export const overrideDefaults = {
   columnNames: undefined,
   shouldTrimFields: undefined,
   grokPattern: undefined,
+  linesToSample: undefined,
 };

x-pack/plugins/ml/public/file_datavisualizer/components/utils/utils.js

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { overrideDefaults } from './overrides';
+import { overrideDefaults, DEFAULT_LINES_TO_SAMPLE } from './overrides';
 import { isEqual } from 'lodash';
 import { ml } from '../../../services/ml_api_service';
 
@@ -86,6 +86,11 @@ export function createUrlOverrides(overrides, originalSettings) {
   if (formattedOverrides.grok_pattern !== '') {
     formattedOverrides.grok_pattern = encodeURIComponent(formattedOverrides.grok_pattern);
   }
+
+  if (formattedOverrides.lines_to_sample === '') {
+    formattedOverrides.lines_to_sample = overrides.linesToSample;
+  }
+
   return formattedOverrides;
 }
 
@@ -93,6 +98,9 @@ export function processResults(results) {
   const timestampFormat = (results.joda_timestamp_formats !== undefined && results.joda_timestamp_formats.length) ?
     results.joda_timestamp_formats[0] : undefined;
 
+  const linesToSample = (results.overrides !== undefined && results.overrides.lines_to_sample !== undefined) ?
+    results.overrides.lines_to_sample : DEFAULT_LINES_TO_SAMPLE;
+
   return {
     format: results.format,
     delimiter: results.delimiter,
@@ -104,6 +112,7 @@ export function processResults(results) {
     charset: results.charset,
     columnNames: results.column_names,
     grokPattern: results.grok_pattern,
+    linesToSample,
   };
 }
 

x-pack/plugins/ml/server/client/elasticsearch_ml.js

@@ -570,7 +570,7 @@ export const elasticsearchJsPlugin = (Client, config, components) => {
     urls: [
       {
         // eslint-disable-next-line max-len
-        fmt: '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>&quote=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>&timestamp_field=<%=timestamp_field%>&timestamp_format=<%=timestamp_format%>',
+        fmt: '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>&quote=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>&timestamp_field=<%=timestamp_field%>&timestamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>',
         req: {
           charset: {
             type: 'string'
@@ -602,6 +602,9 @@ export const elasticsearchJsPlugin = (Client, config, components) => {
           timestamp_format: {
             type: 'string'
           },
+          lines_to_sample: {
+            type: 'string'
+          },
         }
       },
       {