[ML] Adding configurable file size to file data visualizer (#62752)

* [ML] Adding configurable file size to file data visualizer

* updating translated strings

x-pack/plugins/ml/common/constants/file_datavisualizer.ts

@@ -5,6 +5,8 @@
  */
 
 export const MAX_BYTES = 104857600;
+export const ABSOLUTE_MAX_BYTES = MAX_BYTES * 5;
+export const FILE_SIZE_DISPLAY_FORMAT = '0,0.[0] b';
 
 // Value to use in the Elasticsearch index mapping meta data to identify the
 // index as having been created by the ML File Data Visualizer.

x-pack/plugins/ml/common/types/ml_config.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { schema, TypeOf } from '@kbn/config-schema';
+import { MAX_BYTES } from '../constants/file_datavisualizer';
+
+export const configSchema = schema.object({
+  file_data_visualizer: schema.object({
+    max_file_size_bytes: schema.number({ defaultValue: MAX_BYTES }),
+  }),
+});
+
+export type MlConfigType = TypeOf<typeof configSchema>;

x-pack/plugins/ml/public/application/app.tsx

@@ -15,10 +15,14 @@ import { KibanaContextProvider } from '../../../../../src/plugins/kibana_react/p
 import { setDependencyCache, clearCache } from './util/dependency_cache';
 import { setLicenseCache } from './license';
 import { MlSetupDependencies, MlStartDependencies } from '../plugin';
+import { MlConfigType } from '../../common/types/ml_config';
 
 import { MlRouter } from './routing';
 
-type MlDependencies = MlSetupDependencies & MlStartDependencies;
+type MlDependencies = MlSetupDependencies &
+  MlStartDependencies & {
+    mlConfig: MlConfigType;
+  };
 
 interface AppProps {
   coreStart: CoreStart;
@@ -74,6 +78,7 @@ export const renderApp = (
     http: coreStart.http,
     security: deps.security,
     urlGenerators: deps.share.urlGenerators,
+    mlConfig: deps.mlConfig,
   });
 
   const mlLicense = setLicenseCache(deps.licensing);

x-pack/plugins/ml/public/application/datavisualizer/datavisualizer_selector.tsx

@@ -26,6 +26,7 @@ import { isFullLicense } from '../license';
 import { useTimefilter, useMlKibana } from '../contexts/kibana';
 
 import { NavigationMenu } from '../components/navigation_menu';
+import { getMaxBytesFormatted } from './file_based/components/utils';
 
 function startTrialDescription() {
   return (
@@ -59,6 +60,8 @@ export const DatavisualizerSelector: FC = () => {
     licenseManagement.enabled === true &&
     isFullLicense() === false;
 
+  const maxFileSize = getMaxBytesFormatted();
+
   return (
     <Fragment>
       <NavigationMenu tabId="datavisualizer" />
@@ -102,7 +105,8 @@ export const DatavisualizerSelector: FC = () => {
                 description={
                   <FormattedMessage
                     id="xpack.ml.datavisualizer.selector.importDataDescription"
-                    defaultMessage="Import data from a log file. You can upload files up to 100 MB."
+                    defaultMessage="Import data from a log file. You can upload files up to {maxFileSize}."
+                    values={{ maxFileSize }}
                   />
                 }
                 betaBadgeLabel={i18n.translate(

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx

@@ -19,6 +19,7 @@ import {
 } from '@elastic/eui';
 
 import { ExperimentalBadge } from '../experimental_badge';
+import { getMaxBytesFormatted } from '../utils';
 
 export const WelcomeContent: FC = () => {
   const toolTipContent = i18n.translate(
@@ -28,6 +29,8 @@ export const WelcomeContent: FC = () => {
     }
   );
 
+  const maxFileSize = getMaxBytesFormatted();
+
   return (
     <EuiFlexGroup gutterSize="xl" alignItems="center">
       <EuiFlexItem grow={false}>
@@ -117,7 +120,8 @@ export const WelcomeContent: FC = () => {
           <p>
             <FormattedMessage
               id="xpack.ml.fileDatavisualizer.welcomeContent.uploadedFilesAllowedSizeDescription"
-              defaultMessage="You can upload files up to 100 MB."
+              defaultMessage="You can upload files up to {maxFileSize}."
+              values={{ maxFileSize }}
             />
           </p>
         </EuiText>

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js

@@ -19,14 +19,15 @@ import { FileCouldNotBeRead, FileTooLarge } from './file_error_callouts';
 import { EditFlyout } from '../edit_flyout';
 import { ExplanationFlyout } from '../explanation_flyout';
 import { ImportView } from '../import_view';
-import { MAX_BYTES } from '../../../../../../common/constants/file_datavisualizer';
 import {
+  getMaxBytes,
   readFile,
   createUrlOverrides,
   processResults,
   reduceData,
   hasImportPermission,
 } from '../utils';
+
 import { MODE } from './constants';
 
 const UPLOAD_SIZE_MB = 5;
@@ -57,6 +58,7 @@ export class FileDataVisualizerView extends Component {
     this.overrides = {};
     this.previousOverrides = {};
     this.originalSettings = {};
+    this.maxFileUploadBytes = getMaxBytes();
   }
 
   async componentDidMount() {
@@ -93,7 +95,7 @@ export class FileDataVisualizerView extends Component {
   };
 
   async loadFile(file) {
-    if (file.size <= MAX_BYTES) {
+    if (file.size <= this.maxFileUploadBytes) {
       try {
         const fileContents = await readFile(file);
         const data = fileContents.data;
@@ -105,7 +107,6 @@ export class FileDataVisualizerView extends Component {
 
         await this.loadSettings(data);
       } catch (error) {
-        console.error(error);
         this.setState({
           loaded: false,
           loading: false,
@@ -181,8 +182,6 @@ export class FileDataVisualizerView extends Component {
         fileCouldNotBeRead: isRetry,
       });
     } catch (error) {
-      console.error(error);
-
       this.setState({
         results: undefined,
         explanation: undefined,
@@ -287,7 +286,9 @@ export class FileDataVisualizerView extends Component {
 
             {loading && <LoadingPanel />}
 
-            {fileTooLarge && <FileTooLarge fileSize={fileSize} maxFileSize={MAX_BYTES} />}
+            {fileTooLarge && (
+              <FileTooLarge fileSize={fileSize} maxFileSize={this.maxFileUploadBytes} />
+            )}
 
             {fileCouldNotBeRead && loading === false && (
               <React.Fragment>

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_error_callouts.tsx

@@ -11,8 +11,7 @@ import { EuiCallOut, EuiSpacer } from '@elastic/eui';
 
 import numeral from '@elastic/numeral';
 import { ErrorResponse } from '../../../../../../common/types/errors';
-
-const FILE_SIZE_DISPLAY_FORMAT = '0,0.[0] b';
+import { FILE_SIZE_DISPLAY_FORMAT } from '../../../../../../common/constants/file_datavisualizer';
 
 interface FileTooLargeProps {
   fileSize: number;
@@ -81,7 +80,7 @@ interface FileCouldNotBeReadProps {
 }
 
 export const FileCouldNotBeRead: FC<FileCouldNotBeReadProps> = ({ error, loaded }) => {
-  const message = error.body.message;
+  const message = error?.body?.message || '';
   return (
     <EuiCallOut
       title={
@@ -110,7 +109,7 @@ export const FileCouldNotBeRead: FC<FileCouldNotBeReadProps> = ({ error, loaded
 };
 
 export const Explanation: FC<{ error: ErrorResponse }> = ({ error }) => {
-  if (!error.body.attributes?.body?.error?.suppressed?.length) {
+  if (!error?.body?.attributes?.body?.error?.suppressed?.length) {
     return null;
   }
   const reason: string = error.body.attributes.body.error.suppressed[0].reason;

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts

@@ -10,4 +10,6 @@ export {
   processResults,
   readFile,
   reduceData,
+  getMaxBytes,
+  getMaxBytesFormatted,
 } from './utils';

x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts

@@ -5,8 +5,14 @@
  */
 
 import { isEqual } from 'lodash';
+import numeral from '@elastic/numeral';
 import { ml } from '../../../../services/ml_api_service';
 import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer';
+import {
+  ABSOLUTE_MAX_BYTES,
+  FILE_SIZE_DISPLAY_FORMAT,
+} from '../../../../../../common/constants/file_datavisualizer';
+import { getMlConfig } from '../../../../util/dependency_cache';
 
 const DEFAULT_LINES_TO_SAMPLE = 1000;
 
@@ -54,6 +60,15 @@ export function reduceData(data: string, mb: number) {
   return data.length >= size ? data.slice(0, size) : data;
 }
 
+export function getMaxBytes() {
+  const maxBytes = getMlConfig().file_data_visualizer.max_file_size_bytes;
+  return maxBytes < ABSOLUTE_MAX_BYTES ? maxBytes : ABSOLUTE_MAX_BYTES;
+}
+
+export function getMaxBytesFormatted() {
+  return numeral(getMaxBytes()).format(FILE_SIZE_DISPLAY_FORMAT);
+}
+
 export function createUrlOverrides(overrides: InputOverrides, originalSettings: InputOverrides) {
   const formattedOverrides: InputOverrides = {};
   for (const o in overrideDefaults) {

x-pack/plugins/ml/public/application/util/dependency_cache.ts

@@ -23,6 +23,7 @@ import {
 } from 'kibana/public';
 import { SharePluginStart } from 'src/plugins/share/public';
 import { SecurityPluginSetup } from '../../../../security/public';
+import { MlConfigType } from '../../../common/types/ml_config';
 
 export interface DependencyCache {
   timefilter: DataPublicPluginSetup['query']['timefilter'] | null;
@@ -42,6 +43,7 @@ export interface DependencyCache {
   security: SecurityPluginSetup | null;
   i18n: I18nStart | null;
   urlGenerators: SharePluginStart['urlGenerators'] | null;
+  mlConfig: MlConfigType | null;
 }
 
 const cache: DependencyCache = {
@@ -62,6 +64,7 @@ const cache: DependencyCache = {
   security: null,
   i18n: null,
   urlGenerators: null,
+  mlConfig: null,
 };
 
 export function setDependencyCache(deps: Partial<DependencyCache>) {
@@ -82,6 +85,7 @@ export function setDependencyCache(deps: Partial<DependencyCache>) {
   cache.security = deps.security || null;
   cache.i18n = deps.i18n || null;
   cache.urlGenerators = deps.urlGenerators || null;
+  cache.mlConfig = deps.mlConfig || null;
 }
 
 export function getTimefilter() {
@@ -202,6 +206,13 @@ export function getGetUrlGenerator() {
   return cache.urlGenerators.getUrlGenerator;
 }
 
+export function getMlConfig() {
+  if (cache.mlConfig === null) {
+    throw new Error("mlConfig hasn't been initialized");
+  }
+  return cache.mlConfig;
+}
+
 export function clearCache() {
   console.log('clearing dependency cache'); // eslint-disable-line no-console
   Object.keys(cache).forEach(k => {

x-pack/plugins/ml/public/index.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { PluginInitializer } from 'kibana/public';
+import { PluginInitializer, PluginInitializerContext } from 'kibana/public';
 import './index.scss';
 import {
   MlPlugin,
@@ -19,6 +19,6 @@ export const plugin: PluginInitializer<
   MlPluginStart,
   MlSetupDependencies,
   MlStartDependencies
-> = () => new MlPlugin();
+> = (context: PluginInitializerContext) => new MlPlugin(context);
 
 export { MlPluginSetup, MlPluginStart };

x-pack/plugins/ml/public/plugin.ts

@@ -5,7 +5,13 @@
  */
 
 import { i18n } from '@kbn/i18n';
-import { Plugin, CoreStart, CoreSetup, AppMountParameters } from 'kibana/public';
+import {
+  Plugin,
+  CoreStart,
+  CoreSetup,
+  AppMountParameters,
+  PluginInitializerContext,
+} from 'kibana/public';
 import { ManagementSetup } from 'src/plugins/management/public';
 import { SharePluginStart } from 'src/plugins/share/public';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
@@ -19,6 +25,7 @@ import { LicenseManagementUIPluginSetup } from '../../license_management/public'
 import { setDependencyCache } from './application/util/dependency_cache';
 import { PLUGIN_ID, PLUGIN_ICON } from '../common/constants/app';
 import { registerFeature } from './register_feature';
+import { MlConfigType } from '../common/types/ml_config';
 
 export interface MlStartDependencies {
   data: DataPublicPluginStart;
@@ -34,7 +41,10 @@ export interface MlSetupDependencies {
 }
 
 export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
+  constructor(private readonly initializerContext: PluginInitializerContext) {}
+
   setup(core: CoreSetup<MlStartDependencies, MlPluginStart>, pluginsSetup: MlSetupDependencies) {
+    const mlConfig = this.initializerContext.config.get<MlConfigType>();
     core.application.register({
       id: PLUGIN_ID,
       title: i18n.translate('xpack.ml.plugin.title', {
@@ -57,6 +67,7 @@ export class MlPlugin implements Plugin<MlPluginSetup, MlPluginStart> {
             usageCollection: pluginsSetup.usageCollection,
             licenseManagement: pluginsSetup.licenseManagement,
             home: pluginsSetup.home,
+            mlConfig,
           },
           {
             element: params.element,

x-pack/plugins/ml/server/config.ts

@@ -0,0 +1,15 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { PluginConfigDescriptor } from 'kibana/server';
+import { MlConfigType, configSchema } from '../common/types/ml_config';
+
+export const config: PluginConfigDescriptor<MlConfigType> = {
+  exposeToBrowser: {
+    file_data_visualizer: true,
+  },
+  schema: configSchema,
+};

x-pack/plugins/ml/server/index.ts

@@ -9,3 +9,5 @@ import { MlServerPlugin } from './plugin';
 export { MlPluginSetup, MlPluginStart } from './plugin';
 
 export const plugin = (ctx: PluginInitializerContext) => new MlServerPlugin(ctx);
+
+export { config } from './config';

x-pack/plugins/translations/translations/ja-JP.json

@@ -9732,7 +9732,6 @@
     "xpack.ml.datavisualizer.selector.dataVisualizerTitle": "データビジュアライザー",
     "xpack.ml.datavisualizer.selector.experimentalBadgeLabel": "実験的",
     "xpack.ml.datavisualizer.selector.experimentalBadgeTooltipLabel": "実験的機能。フィードバックをお待ちしています。",
-    "xpack.ml.datavisualizer.selector.importDataDescription": "ログファイルからデータをインポートします。最大 100 MB のファイルをアップロードできます。",
     "xpack.ml.datavisualizer.selector.importDataTitle": "データのインポート",
     "xpack.ml.datavisualizer.selector.selectIndexButtonLabel": "インデックスを選択",
     "xpack.ml.datavisualizer.selector.selectIndexPatternDescription": "既存の Elasticsearch インデックスのデータを可視化します。",
@@ -9983,7 +9982,6 @@
     "xpack.ml.fileDatavisualizer.welcomeContent.logFilesWithCommonFormatDescription": "タイムスタンプの一般的フォーマットのログファイル",
     "xpack.ml.fileDatavisualizer.welcomeContent.newlineDelimitedJsonDescription": "改行区切りの JSON",
     "xpack.ml.fileDatavisualizer.welcomeContent.supportedFileFormatDescription": "ファイルデータビジュアライザーはこれらのファイル形式をサポートしています:",
-    "xpack.ml.fileDatavisualizer.welcomeContent.uploadedFilesAllowedSizeDescription": "最大 100 MB のファイルをアップロードできます。",
     "xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileDescription": "ファイルデータビジュアライザーは、ログファイルのフィールドとメトリックの理解に役立ちます。ファイルをアップロードして、データを分析し、 Elasticsearch インデックスにインポートするか選択できます。",
     "xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle": "ログファイルのデータを可視化 {experimentalBadge}",
     "xpack.ml.formatters.metricChangeDescription.actualSameAsTypicalDescription": "実際値が通常値と同じ",

x-pack/plugins/translations/translations/zh-CN.json

@@ -9735,7 +9735,6 @@
     "xpack.ml.datavisualizer.selector.dataVisualizerTitle": "数据可视化工具",
     "xpack.ml.datavisualizer.selector.experimentalBadgeLabel": "实验性",
     "xpack.ml.datavisualizer.selector.experimentalBadgeTooltipLabel": "实验功能。我们很乐意听取您的反馈意见。",
-    "xpack.ml.datavisualizer.selector.importDataDescription": "从日志文件导入数据。您可以上传最大 100 MB 的文件。",
     "xpack.ml.datavisualizer.selector.importDataTitle": "导入数据",
     "xpack.ml.datavisualizer.selector.selectIndexButtonLabel": "选择索引",
     "xpack.ml.datavisualizer.selector.selectIndexPatternDescription": "可视化现有 Elasticsearch 索引中的数据。",
@@ -9986,7 +9985,6 @@
     "xpack.ml.fileDatavisualizer.welcomeContent.logFilesWithCommonFormatDescription": "具有时间戳通用格式的日志文件",
     "xpack.ml.fileDatavisualizer.welcomeContent.newlineDelimitedJsonDescription": "换行符分隔的 JSON",
     "xpack.ml.fileDatavisualizer.welcomeContent.supportedFileFormatDescription": "File Data Visualizer 支持以下文件格式：",
-    "xpack.ml.fileDatavisualizer.welcomeContent.uploadedFilesAllowedSizeDescription": "您可以上传最大 100 MB 的文件。",
     "xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileDescription": "File Data Visualizer 可帮助您理解日志文件中的字段和指标。上传文件、分析文件数据，然后选择是否将数据导入 Elasticsearch 索引。",
     "xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle": "可视化来自日志文件的数据 {experimentalBadge}",
     "xpack.ml.formatters.metricChangeDescription.actualSameAsTypicalDescription": "实际上与典型模式相同",