From 62f82c96f6242964154bc828069a6d0344c3020a Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Wed, 31 Oct 2018 18:06:49 +0000 Subject: [PATCH] [ML] Fixing issues when trying to import non-timestamp data (#24894) * [ML] Fixing issues when trying to import non-timestamp data * making results link time range optional * small cleanup after fixing merge conflict --- .../import_progress/import_progress.js | 23 ++-- .../import_summary/import_summary.js | 32 ++++-- .../components/import_view/import_view.js | 104 ++++++++++++------ .../import_view/importer/importer.js | 6 +- .../components/results_links/results_links.js | 11 +- .../file_data_visualizer/import_data.js | 13 +-- 6 files changed, 127 insertions(+), 62 deletions(-) diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_progress/import_progress.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_progress/import_progress.js index ec447c851dbc..75e3eb4b97a3 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_progress/import_progress.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/import_progress/import_progress.js @@ -30,6 +30,7 @@ export function ImportProgress({ statuses }) { uploadProgress, uploadStatus, createIndexPattern, + createPipeline, } = statuses; let statusInfo = null; @@ -90,6 +91,8 @@ export function ImportProgress({ statuses }) { if (createIndexPattern === true) { createIndexPatternTitle = 'Creating index pattern'; statusInfo = (

Creating index pattern

); + } else { + statusInfo = null; } } if (completedStep >= 5) { @@ -112,22 +115,26 @@ export function ImportProgress({ statuses }) { status: indexCreatedStatus, onClick: () => {}, }, - { - title: createIngestPipelineTitle, - isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE), - isComplete: (ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE), - status: ingestPipelineCreatedStatus, - onClick: () => {}, - }, { title: uploadingDataTitle, - isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE && ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE), + isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE && + (createPipeline === false || (createPipeline === true && ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE))), isComplete: (uploadStatus === IMPORT_STATUS.COMPLETE), status: uploadStatus, onClick: () => {}, } ]; + if (createPipeline === true) { + steps.splice(2, 0, { + title: createIngestPipelineTitle, + isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE), + isComplete: (ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE), + status: ingestPipelineCreatedStatus, + onClick: () => {}, + }); + } + if (createIndexPattern === true) { steps.push({ title: createIndexPatternTitle, diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_summary/import_summary.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_summary/import_summary.js index c727a6f30198..923cd1fe0026 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_summary/import_summary.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/import_summary/import_summary.js @@ -20,13 +20,17 @@ export function ImportSummary({ ingestPipelineId, docCount, importFailures, + createIndexPattern, + createPipeline, }) { const items = createDisplayItems( index, indexPattern, ingestPipelineId, docCount, - importFailures + importFailures, + createIndexPattern, + createPipeline ); return ( @@ -90,27 +94,35 @@ function createDisplayItems( indexPattern, ingestPipelineId, docCount, - importFailures + importFailures, + createIndexPattern, + createPipeline ) { const items = [ { title: 'Index', description: index, }, - { - title: 'Index pattern', - description: indexPattern, - }, - { - title: 'Ingest pipeline', - description: ingestPipelineId, - }, { title: 'Documents ingested', description: docCount - ((importFailures && importFailures.length) || 0), } ]; + if (createPipeline) { + items.splice(1, 0, { + title: 'Ingest pipeline', + description: ingestPipelineId, + }); + } + + if (createIndexPattern) { + items.splice(1, 0, { + title: 'Index pattern', + description: indexPattern, + }); + } + if (importFailures && importFailures.length > 0) { items.push({ title: 'Failed documents', diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js index 9c360a92f0dc..a722b11b9945 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js @@ -57,6 +57,7 @@ const DEFAULT_STATE = { indexPatternNames: [], indexNameError: '', indexPatternNameError: '', + timeFieldName: undefined, }; export class ImportView extends Component { @@ -85,8 +86,16 @@ export class ImportView extends Component { // TODO - sort this function out. it's a mess async import() { - const { fileContents, results } = this.props; + const { + fileContents, + results, + indexPatterns, + kibanaConfig, + showBottomBar, + } = this.props; + const { format } = results; + let { timeFieldName } = this.state; const { index, indexPattern, @@ -108,14 +117,29 @@ export class ImportView extends Component { this.props.hideBottomBar(); setTimeout(async () => { let success = false; + const createPipeline = (pipelineString !== ''); let indexCreationSettings = {}; try { + const settings = JSON.parse(indexSettingsString); + const mappings = JSON.parse(mappingsString); indexCreationSettings = { - settings: JSON.parse(indexSettingsString), - mappings: JSON.parse(mappingsString), - pipeline: JSON.parse(pipelineString), + settings, + mappings, }; + if (createPipeline) { + indexCreationSettings.pipeline = JSON.parse(pipelineString); + } + + // if an @timestamp field has been added to the + // mappings, use this field as the time field. + // This relies on the field being populated by + // the ingest pipeline on ingest + if (mappings[DEFAULT_TIME_FIELD] !== undefined) { + timeFieldName = DEFAULT_TIME_FIELD; + this.setState({ timeFieldName }); + } + success = true; } catch (error) { success = false; @@ -146,15 +170,19 @@ export class ImportView extends Component { indexCreatedStatus: indexCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, }); - const pipelineCreated = (initializeImportResp.pipelineId !== undefined); - if (indexCreated) { - this.setState({ - ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, - ingestPipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', - }); + if (createPipeline) { + const pipelineCreated = (initializeImportResp.pipelineId !== undefined); + if (indexCreated) { + this.setState({ + ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, + ingestPipelineId: pipelineCreated ? initializeImportResp.pipelineId : '', + }); + } + success = (indexCreated && pipelineCreated); + } else { + success = indexCreated; } - success = (indexCreated && pipelineCreated); if (success) { const importId = initializeImportResp.id; @@ -167,21 +195,23 @@ export class ImportView extends Component { docCount: importResp.docCount, }); - if (success && createIndexPattern) { - const indexPatternName = (indexPattern === '') ? index : indexPattern; - - const indexPatternResp = await createKibanaIndexPattern( - indexPatternName, - this.props.indexPatterns, - this.props.kibanaConfig, - ); - success = indexPatternResp.success; - this.setState({ - indexPatternCreatedStatus: indexPatternResp.success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, - indexPatternId: indexPatternResp.id, - }); - if (indexPatternResp.success === false) { - errors.push(indexPatternResp.error); + if (success) { + if (createIndexPattern) { + const indexPatternName = (indexPattern === '') ? index : indexPattern; + const indexPatternResp = await createKibanaIndexPattern( + indexPatternName, + indexPatterns, + timeFieldName, + kibanaConfig, + ); + success = indexPatternResp.success; + this.setState({ + indexPatternCreatedStatus: indexPatternResp.success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED, + indexPatternId: indexPatternResp.id, + }); + if (indexPatternResp.success === false) { + errors.push(indexPatternResp.error); + } } } else { errors.push(importResp.error); @@ -193,7 +223,8 @@ export class ImportView extends Component { } } - this.props.showBottomBar(); + showBottomBar(); + this.setState({ importing: false, imported: success, @@ -301,8 +332,11 @@ export class ImportView extends Component { pipelineString, indexNameError, indexPatternNameError, + timeFieldName, } = this.state; + const createPipeline = (pipelineString !== ''); + const statuses = { reading, readStatus, @@ -312,6 +346,7 @@ export class ImportView extends Component { uploadProgress, uploadStatus, createIndexPattern, + createPipeline, }; const disableImport = ( @@ -399,14 +434,16 @@ export class ImportView extends Component { ingestPipelineId={ingestPipelineId} docCount={docCount} importFailures={importFailures} + createIndexPattern={createIndexPattern} + createPipeline={createPipeline} /> } @@ -433,7 +470,7 @@ export class ImportView extends Component { } } -async function createKibanaIndexPattern(indexPatternName, indexPatterns, kibanaConfig, timeFieldName = DEFAULT_TIME_FIELD) { +async function createKibanaIndexPattern(indexPatternName, indexPatterns, timeFieldName, kibanaConfig) { try { const emptyPattern = await indexPatterns.get(); @@ -467,13 +504,16 @@ async function createKibanaIndexPattern(indexPatternName, indexPatterns, kibanaC function getDefaultState(state, results) { const indexSettingsString = (state.indexSettingsString === '') ? '{}' : state.indexSettingsString; const mappingsString = (state.mappingsString === '') ? JSON.stringify(results.mappings, null, 2) : state.mappingsString; - const pipelineString = (state.pipelineString === '') ? JSON.stringify(results.ingest_pipeline, null, 2) : state.pipelineString; + const pipelineString = (state.pipelineString === '' && results.ingest_pipeline !== undefined) ? + JSON.stringify(results.ingest_pipeline, null, 2) : state.pipelineString; + const timeFieldName = results.timestamp_field; return { ... DEFAULT_STATE, indexSettingsString, mappingsString, pipelineString, + timeFieldName, }; } diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/importer.js b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/importer.js index cb1b47587481..dcd46262b096 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/importer.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/importer.js @@ -28,10 +28,12 @@ export class Importer { const pipeline = this.pipeline; updatePipelineTimezone(pipeline); - const ingestPipeline = { + // if no pipeline has been supplied, + // send an empty object + const ingestPipeline = (pipeline !== undefined) ? { id: `${index}-pipeline`, pipeline, - }; + } : {}; const createIndexResp = await ml.fileDatavisualizer.import({ id: undefined, diff --git a/x-pack/plugins/ml/public/file_datavisualizer/components/results_links/results_links.js b/x-pack/plugins/ml/public/file_datavisualizer/components/results_links/results_links.js index 154cb397d991..b88c87c21758 100644 --- a/x-pack/plugins/ml/public/file_datavisualizer/components/results_links/results_links.js +++ b/x-pack/plugins/ml/public/file_datavisualizer/components/results_links/results_links.js @@ -36,7 +36,11 @@ export class ResultsLinks extends Component { } componentDidMount() { - this.updateTimeValues(); + // if this data has a time field, + // find the start and end times + if (this.props.timeFieldName !== undefined) { + this.updateTimeValues(); + } } componentWillUnmount() { @@ -71,6 +75,7 @@ export class ResultsLinks extends Component { render() { const { indexPatternId, + timeFieldName, } = this.props; const { @@ -78,7 +83,7 @@ export class ResultsLinks extends Component { to, } = this.state; - const _g = `&_g=(time:(from:'${from}',mode:quick,to:'${to}'))`; + const _g = (this.props.timeFieldName !== undefined) ? `&_g=(time:(from:'${from}',mode:quick,to:'${to}'))` : ''; return ( @@ -91,7 +96,7 @@ export class ResultsLinks extends Component { /> - {(isFullLicense() === true) && + {(isFullLicense() === true && timeFieldName !== undefined) && } diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js index 983dd4c2670d..89d209f5ad64 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js @@ -13,10 +13,6 @@ export function importDataProvider(callWithRequest) { try { - if (ingestPipeline === undefined || ingestPipeline.id === undefined) { - throw 'No ingest pipeline id specified'; - } - const { id: pipelineId, pipeline, @@ -29,9 +25,12 @@ export function importDataProvider(callWithRequest) { await createIndex(index, settings, mappings); createdIndex = index; - const success = await createPipeline(pipelineId, pipeline); - if (success.acknowledged !== true) { - throw success; + // create the pipeline if one has been supplied + if (pipelineId !== undefined) { + const success = await createPipeline(pipelineId, pipeline); + if (success.acknowledged !== true) { + throw success; + } } createdPipelineId = pipelineId;