maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Fixing issues when trying to import non-timestamp data (#24894)

Browse source 
* [ML] Fixing issues when trying to import non-timestamp data

* making results link time range optional

* small cleanup after fixing merge conflict
This commit is contained in:
James Gowdy 2018-10-31 18:06:49 +00:00 committed by GitHub
parent 27aea12c08
commit 62f82c96f6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 127 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
x-pack/plugins/ml/public/file_datavisualizer/components/import_progress/import_progress.js
View file

@ -30,6 +30,7 @@ export function ImportProgress({ statuses }) {
uploadProgress,
uploadStatus,
createIndexPattern,
createPipeline,
} = statuses;
let statusInfo = null;
@ -90,6 +91,8 @@ export function ImportProgress({ statuses }) {
if (createIndexPattern === true) {
createIndexPatternTitle = 'Creating index pattern';
statusInfo = (<p>Creating index pattern</p>);
} else {
statusInfo = null;
}
}
if (completedStep >= 5) {
@ -112,22 +115,26 @@ export function ImportProgress({ statuses }) {
status: indexCreatedStatus,
onClick: () => {},
},
{
title: createIngestPipelineTitle,
isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE),
isComplete: (ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE),
status: ingestPipelineCreatedStatus,
onClick: () => {},
},
{
title: uploadingDataTitle,
isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE && ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE),
isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE &&
(createPipeline === false || (createPipeline === true && ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE))),
isComplete: (uploadStatus === IMPORT_STATUS.COMPLETE),
status: uploadStatus,
onClick: () => {},
}
];
if (createPipeline === true) {
steps.splice(2, 0, {
title: createIngestPipelineTitle,
isSelected: (indexCreatedStatus === IMPORT_STATUS.COMPLETE),
isComplete: (ingestPipelineCreatedStatus === IMPORT_STATUS.COMPLETE),
status: ingestPipelineCreatedStatus,
onClick: () => {},
});
}
if (createIndexPattern === true) {
steps.push({
title: createIndexPatternTitle,

32
x-pack/plugins/ml/public/file_datavisualizer/components/import_summary/import_summary.js
View file

@ -20,13 +20,17 @@ export function ImportSummary({
ingestPipelineId,
docCount,
importFailures,
createIndexPattern,
createPipeline,
}) {
const items = createDisplayItems(
index,
indexPattern,
ingestPipelineId,
docCount,
importFailures
importFailures,
createIndexPattern,
createPipeline
);
return (
@ -90,27 +94,35 @@ function createDisplayItems(
indexPattern,
ingestPipelineId,
docCount,
importFailures
importFailures,
createIndexPattern,
createPipeline
) {
const items = [
{
title: 'Index',
description: index,
},
{
title: 'Index pattern',
description: indexPattern,
},
{
title: 'Ingest pipeline',
description: ingestPipelineId,
},
{
title: 'Documents ingested',
description: docCount - ((importFailures && importFailures.length) || 0),
}
];
if (createPipeline) {
items.splice(1, 0, {
title: 'Ingest pipeline',
description: ingestPipelineId,
});
}
if (createIndexPattern) {
items.splice(1, 0, {
title: 'Index pattern',
description: indexPattern,
});
}
if (importFailures && importFailures.length > 0) {
items.push({
title: 'Failed documents',

104
x-pack/plugins/ml/public/file_datavisualizer/components/import_view/import_view.js
View file

@ -57,6 +57,7 @@ const DEFAULT_STATE = {
indexPatternNames: [],
indexNameError: '',
indexPatternNameError: '',
timeFieldName: undefined,
};
export class ImportView extends Component {
@ -85,8 +86,16 @@ export class ImportView extends Component {
// TODO - sort this function out. it's a mess
async import() {
const { fileContents, results } = this.props;
const {
fileContents,
results,
indexPatterns,
kibanaConfig,
showBottomBar,
} = this.props;
const { format } = results;
let { timeFieldName } = this.state;
const {
index,
indexPattern,
@ -108,14 +117,29 @@ export class ImportView extends Component {
this.props.hideBottomBar();
setTimeout(async () => {
let success = false;
const createPipeline = (pipelineString !== '');
let indexCreationSettings = {};
try {
const settings = JSON.parse(indexSettingsString);
const mappings = JSON.parse(mappingsString);
indexCreationSettings = {
settings: JSON.parse(indexSettingsString),
mappings: JSON.parse(mappingsString),
pipeline: JSON.parse(pipelineString),
settings,
mappings,
};
if (createPipeline) {
indexCreationSettings.pipeline = JSON.parse(pipelineString);
}
// if an @timestamp field has been added to the
// mappings, use this field as the time field.
// This relies on the field being populated by
// the ingest pipeline on ingest
if (mappings[DEFAULT_TIME_FIELD] !== undefined) {
timeFieldName = DEFAULT_TIME_FIELD;
this.setState({ timeFieldName });
}
success = true;
} catch (error) {
success = false;
@ -146,15 +170,19 @@ export class ImportView extends Component {
indexCreatedStatus: indexCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED,
});
const pipelineCreated = (initializeImportResp.pipelineId !== undefined);
if (indexCreated) {
this.setState({
ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED,
ingestPipelineId: pipelineCreated ? initializeImportResp.pipelineId : '',
});
if (createPipeline) {
const pipelineCreated = (initializeImportResp.pipelineId !== undefined);
if (indexCreated) {
this.setState({
ingestPipelineCreatedStatus: pipelineCreated ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED,
ingestPipelineId: pipelineCreated ? initializeImportResp.pipelineId : '',
});
}
success = (indexCreated && pipelineCreated);
} else {
success = indexCreated;
}
success = (indexCreated && pipelineCreated);
if (success) {
const importId = initializeImportResp.id;
@ -167,21 +195,23 @@ export class ImportView extends Component {
docCount: importResp.docCount,
});
if (success && createIndexPattern) {
const indexPatternName = (indexPattern === '') ? index : indexPattern;
const indexPatternResp = await createKibanaIndexPattern(
indexPatternName,
this.props.indexPatterns,
this.props.kibanaConfig,
);
success = indexPatternResp.success;
this.setState({
indexPatternCreatedStatus: indexPatternResp.success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED,
indexPatternId: indexPatternResp.id,
});
if (indexPatternResp.success === false) {
errors.push(indexPatternResp.error);
if (success) {
if (createIndexPattern) {
const indexPatternName = (indexPattern === '') ? index : indexPattern;
const indexPatternResp = await createKibanaIndexPattern(
indexPatternName,
indexPatterns,
timeFieldName,
kibanaConfig,
);
success = indexPatternResp.success;
this.setState({
indexPatternCreatedStatus: indexPatternResp.success ? IMPORT_STATUS.COMPLETE : IMPORT_STATUS.FAILED,
indexPatternId: indexPatternResp.id,
});
if (indexPatternResp.success === false) {
errors.push(indexPatternResp.error);
}
}
} else {
errors.push(importResp.error);
@ -193,7 +223,8 @@ export class ImportView extends Component {
}
}
this.props.showBottomBar();
showBottomBar();
this.setState({
importing: false,
imported: success,
@ -301,8 +332,11 @@ export class ImportView extends Component {
pipelineString,
indexNameError,
indexPatternNameError,
timeFieldName,
} = this.state;
const createPipeline = (pipelineString !== '');
const statuses = {
reading,
readStatus,
@ -312,6 +346,7 @@ export class ImportView extends Component {
uploadProgress,
uploadStatus,
createIndexPattern,
createPipeline,
};
const disableImport = (
@ -399,14 +434,16 @@ export class ImportView extends Component {
ingestPipelineId={ingestPipelineId}
docCount={docCount}
importFailures={importFailures}
createIndexPattern={createIndexPattern}
createPipeline={createPipeline}
/>
<EuiSpacer size="l" />
<ResultsLinks
index={(index)}
indexPatternId={(indexPatternId)}
timeFieldName={DEFAULT_TIME_FIELD}
index={index}
indexPatternId={indexPatternId}
timeFieldName={timeFieldName}
/>
</React.Fragment>
}
@ -433,7 +470,7 @@ export class ImportView extends Component {
}
}
async function createKibanaIndexPattern(indexPatternName, indexPatterns, kibanaConfig, timeFieldName = DEFAULT_TIME_FIELD) {
async function createKibanaIndexPattern(indexPatternName, indexPatterns, timeFieldName, kibanaConfig) {
try {
const emptyPattern = await indexPatterns.get();
@ -467,13 +504,16 @@ async function createKibanaIndexPattern(indexPatternName, indexPatterns, kibanaC
function getDefaultState(state, results) {
const indexSettingsString = (state.indexSettingsString === '') ? '{}' : state.indexSettingsString;
const mappingsString = (state.mappingsString === '') ? JSON.stringify(results.mappings, null, 2) : state.mappingsString;
const pipelineString = (state.pipelineString === '') ? JSON.stringify(results.ingest_pipeline, null, 2) : state.pipelineString;
const pipelineString = (state.pipelineString === '' && results.ingest_pipeline !== undefined) ?
JSON.stringify(results.ingest_pipeline, null, 2) : state.pipelineString;
const timeFieldName = results.timestamp_field;
return {
... DEFAULT_STATE,
indexSettingsString,
mappingsString,
pipelineString,
timeFieldName,
};
}

6
x-pack/plugins/ml/public/file_datavisualizer/components/import_view/importer/importer.js
View file

@ -28,10 +28,12 @@ export class Importer {
const pipeline = this.pipeline;
updatePipelineTimezone(pipeline);
const ingestPipeline = {
// if no pipeline has been supplied,
// send an empty object
const ingestPipeline = (pipeline !== undefined) ? {
id: `${index}-pipeline`,
pipeline,
};
} : {};
const createIndexResp = await ml.fileDatavisualizer.import({
id: undefined,

11
x-pack/plugins/ml/public/file_datavisualizer/components/results_links/results_links.js
View file

@ -36,7 +36,11 @@ export class ResultsLinks extends Component {
}
componentDidMount() {
this.updateTimeValues();
// if this data has a time field,
// find the start and end times
if (this.props.timeFieldName !== undefined) {
this.updateTimeValues();
}
}
componentWillUnmount() {
@ -71,6 +75,7 @@ export class ResultsLinks extends Component {
render() {
const {
indexPatternId,
timeFieldName,
} = this.props;
const {
@ -78,7 +83,7 @@ export class ResultsLinks extends Component {
to,
} = this.state;
const _g = `&_g=(time:(from:'${from}',mode:quick,to:'${to}'))`;
const _g = (this.props.timeFieldName !== undefined) ? `&_g=(time:(from:'${from}',mode:quick,to:'${to}'))` : '';
return (
<EuiFlexGroup gutterSize="l">
@ -91,7 +96,7 @@ export class ResultsLinks extends Component {
/>
</EuiFlexItem>
{(isFullLicense() === true) &&
{(isFullLicense() === true && timeFieldName !== undefined) &&
<EuiFlexItem>
<EuiCard
icon={<EuiIcon size="xxl" type={`machineLearningApp`} />}

13
x-pack/plugins/ml/server/models/file_data_visualizer/import_data.js
View file

@ -13,10 +13,6 @@ export function importDataProvider(callWithRequest) {
try {
if (ingestPipeline === undefined || ingestPipeline.id === undefined) {
throw 'No ingest pipeline id specified';
}
const {
id: pipelineId,
pipeline,
@ -29,9 +25,12 @@ export function importDataProvider(callWithRequest) {
await createIndex(index, settings, mappings);
createdIndex = index;
const success = await createPipeline(pipelineId, pipeline);
if (success.acknowledged !== true) {
throw success;
// create the pipeline if one has been supplied
if (pipelineId !== undefined) {
const success = await createPipeline(pipelineId, pipeline);
if (success.acknowledged !== true) {
throw success;
}
}
createdPipelineId = pipelineId;