Merge pull request #6845 from Bargs/ingest/uploadUI

Add Data - CSV Upload UI
This commit is contained in:
Joe Fleming 2016-06-01 14:51:55 -07:00
commit 94cc728851
29 changed files with 855 additions and 73 deletions

View file

@ -132,6 +132,7 @@
"moment": "2.10.6",
"moment-timezone": "0.4.1",
"node-uuid": "1.4.7",
"papaparse": "4.1.2",
"raw-loader": "0.5.1",
"request": "2.61.0",
"rimraf": "2.4.3",

View file

@ -62,6 +62,7 @@
ng-attr-placeholder="{{index.defaultName}}"
ng-model-options="{ updateOn: 'default blur', debounce: {'default': 2500, 'blur': 0} }"
validate-index-name
allow-wildcard
name="name"
required
type="text"

View file

@ -0,0 +1,63 @@
<file-upload ng-if="!wizard.file" on-locate="wizard.file = file" upload-selector="button.upload">
<h2><em>Pick a CSV file to get started.</em>
Please follow the instructions below.
</h2>
<div class="upload-wizard-file-upload-container">
<div class="upload-instructions">Drop your file here</div>
<div class="upload-instructions-separator">or</div>
<button class="btn btn-primary btn-lg controls upload" ng-click>
Select File
</button>
<div>Maximum upload file size: 1 GB</div>
</div>
</file-upload>
<div class="upload-wizard-file-preview-container" ng-if="wizard.file">
<h2><em>Review the sample below.</em>
Click next if it looks like we parsed your file correctly.
</h2>
<div ng-if="!!wizard.formattedErrors.length" class="alert alert-danger parse-error">
<ul>
<li ng-repeat="error in wizard.formattedErrors track by $index">{{ error }}</li>
</ul>
</div>
<div ng-if="!!wizard.formattedWarnings.length" class="alert alert-warning">
<ul>
<li ng-repeat="warning in wizard.formattedWarnings track by $index">{{ warning }}</li>
</ul>
</div>
<div class="advanced-options form-inline">
<span class="form-group">
<label>Delimiter</label>
<select ng-model="wizard.parseOptions.delimiter"
ng-options="option.value as option.label for option in wizard.delimiterOptions"
class="form-control">
</select>
</span>
<span class="form-group">
<label>Filename:</label>
{{ wizard.file.name }}
</span>
</div>
<div class="preview">
<table class="table table-condensed">
<thead>
<tr>
<th ng-repeat="col in wizard.columns track by $index">
<span title="{{ col }}">{{ col | limitTo:12 }}{{ col.length > 12 ? '...' : '' }}</span>
</th>
</tr>
</thead>
<tbody>
<tr ng-repeat="row in wizard.rows">
<td ng-repeat="cell in row track by $index">{{ cell }}</td>
</tr>
</tbody>
</table>
</div>
</div>

View file

@ -0,0 +1,132 @@
import _ from 'lodash';
import Papa from 'papaparse';
import modules from 'ui/modules';
import template from './parse_csv_step.html';
import './styles/_add_data_parse_csv_step.less';
modules.get('apps/settings')
.directive('parseCsvStep', function () {
return {
restrict: 'E',
template: template,
scope: {
file: '=',
parseOptions: '=',
samples: '='
},
bindToController: true,
controllerAs: 'wizard',
controller: function ($scope) {
const maxSampleRows = 10;
const maxSampleColumns = 20;
this.delimiterOptions = [
{
label: 'comma',
value: ','
},
{
label: 'tab',
value: '\t'
},
{
label: 'space',
value: ' '
},
{
label: 'semicolon',
value: ';'
},
{
label: 'pipe',
value: '|'
}
];
this.parse = () => {
if (!this.file) return;
let row = 1;
let rows = [];
let data = [];
const config = _.assign(
{
header: true,
dynamicTyping: true,
step: (results, parser) => {
if (row > maxSampleRows) {
parser.abort();
// The complete callback isn't automatically called if parsing is manually aborted
config.complete();
return;
}
if (row === 1) {
// Collect general information on the first pass
if (results.meta.fields.length > _.uniq(results.meta.fields).length) {
this.formattedErrors.push('Column names must be unique');
}
_.forEach(results.meta.fields, (field) => {
if (_.isEmpty(field)) {
this.formattedErrors.push('Column names must not be blank');
}
});
if (results.meta.fields.length > maxSampleColumns) {
this.formattedWarnings.push(`Preview truncated to ${maxSampleColumns} columns`);
}
this.columns = results.meta.fields.slice(0, maxSampleColumns);
this.parseOptions = _.defaults({}, this.parseOptions, {delimiter: results.meta.delimiter});
}
this.formattedErrors = _.map(results.errors, (error) => {
return `${error.type} at line ${row + 1} - ${error.message}`;
});
data = data.concat(results.data);
rows = rows.concat(_.map(results.data, (row) => {
return _.map(this.columns, (columnName) => {
return row[columnName];
});
}));
++row;
},
complete: () => {
$scope.$apply(() => {
this.rows = rows;
if (_.isUndefined(this.formattedErrors) || _.isEmpty(this.formattedErrors)) {
this.samples = data;
}
else {
delete this.samples;
}
});
}
},
this.parseOptions
);
Papa.parse(this.file, config);
};
$scope.$watch('wizard.parseOptions', (newValue, oldValue) => {
// Delimiter is auto-detected in the first run of the parse function, so we don't want to
// re-parse just because it's being initialized.
if (!_.isUndefined(oldValue)) {
this.parse();
}
}, true);
$scope.$watch('wizard.file', () => {
delete this.rows;
delete this.columns;
delete this.formattedErrors;
this.formattedWarnings = [];
this.parse();
});
}
};
});

View file

@ -0,0 +1,64 @@
@import (reference) "../../../styles/_add_data_wizard";
.upload-wizard-file-upload-container {
min-height: 300px;
display: flex;
flex-direction: column;
justify-content: center;
background-color: @settings-add-data-wizard-form-control-bg;
border: @settings-add-data-wizard-parse-csv-container-border 1px dashed;
text-align: center;
.upload-instructions {
font-size: 2em;
}
.upload-instructions-separator {
margin: 15px 0;
}
button {
width: inherit;
}
button.upload {
align-self: center;
margin-bottom: 15px;
}
}
.upload-wizard-file-preview-container {
.preview {
overflow: auto;
max-height: 500px;
border: @settings-add-data-wizard-parse-csv-container-border 1px solid;
table {
margin-bottom: 0;
.table-striped()
}
}
.parse-error {
margin-top: 2em;
}
.advanced-options {
display: flex;
align-items: center;
.form-group {
display: flex;
align-items: center;
padding-right: 15px;
label {
padding-right: 8px;
margin-bottom: 0;
}
}
padding-bottom: 10px;
}
}

View file

@ -3,31 +3,48 @@
fields can be changed if we got it wrong!
</h2>
<div class="pattern-review form-inline">
<div ng-show="reviewStep.errors.length" class="alert alert-danger">
<div ng-repeat="error in reviewStep.errors">{{ error }}</div>
<form name="reviewStep.form">
<div class="pattern-review form-inline">
<div ng-show="reviewStep.errors.length" class="alert alert-danger">
<div ng-repeat="error in reviewStep.errors">{{ error }}</div>
</div>
<div class="alert alert-danger"
ng-show="reviewStep.form.pattern.$dirty && reviewStep.form.pattern.$error.lowercase">
Index names must be all lowercase
</div>
<div class="alert alert-danger"
ng-show="reviewStep.form.pattern.$dirty && reviewStep.form.pattern.$error.indexNameInput">
An index name must not be empty and cannot contain whitespace or any of the following characters: ", *, \, <, |, ,, >, /, ?
</div>
<label>{{ reviewStep.patternInput.label }}</label>
<span id="pattern-help" class="help-block">{{ reviewStep.patternInput.helpText }}</span>
<input name="pattern" ng-model="reviewStep.indexPattern.id"
class="pattern-input form-control"
novalidate
required
validate-index-name
validate-lowercase
placeholder="{{reviewStep.patternInput.placeholder}}"
aria-describedby="pattern-help"/>
<label>
<input ng-model="reviewStep.isTimeBased" type="checkbox"/>
time based
</label>
<label ng-if="reviewStep.isTimeBased" class="time-field-input">
Time Field
<select ng-model="reviewStep.indexPattern.timeFieldName" name="time_field_name" class="form-control">
<option ng-repeat="field in reviewStep.dateFields" value="{{field}}">
{{field}}
</option>
</select>
</label>
</div>
<label>Index name or pattern</label>
<span id="pattern-help" class="help-block">Patterns allow you to define dynamic index names using * as a wildcard. Example: filebeat-*</span>
<input ng-model="reviewStep.indexPattern.id" class="pattern-input form-control" aria-describedby="pattern-help"/>
<label>
<input ng-model="reviewStep.isTimeBased" type="checkbox"/>
time based
</label>
<label ng-if="reviewStep.isTimeBased" class="time-field-input">
Time Field
<select ng-model="reviewStep.indexPattern.timeFieldName" name="time_field_name" class="form-control">
<option ng-repeat="field in reviewStep.dateFields" value="{{field}}">
{{field}}
</option>
</select>
</label>
</div>
<paginated-table
class="pattern-review-field-table"
columns="reviewStep.columns"
rows="reviewStep.rows"
per-page="10">
</paginated-table>
<paginated-table
class="pattern-review-field-table"
columns="reviewStep.columns"
rows="reviewStep.rows"
per-page="10">
</paginated-table>
</form>

View file

@ -6,6 +6,7 @@ import isGeoPointObject from './lib/is_geo_point_object';
import forEachField from './lib/for_each_field';
import './styles/_add_data_pattern_review_step.less';
import moment from 'moment';
import '../../../../../../../../ui/public/directives/validate_lowercase';
function pickDefaultTimeFieldName(dateFields) {
if (_.isEmpty(dateFields)) {
@ -26,7 +27,8 @@ modules.get('apps/settings')
scope: {
indexPattern: '=',
pipeline: '=',
sampleDoc: '='
sampleDoc: '=',
defaultIndexInput: '='
},
controllerAs: 'reviewStep',
bindToController: true,
@ -34,6 +36,17 @@ modules.get('apps/settings')
this.errors = [];
const sampleFields = {};
this.patternInput = {
label: 'Index name',
helpText: 'The name of the Elasticsearch index you want to create for your data.',
defaultValue: '',
placeholder: 'Name'
};
if (this.defaultIndexInput) {
this.patternInput.defaultValue = this.defaultIndexInput;
}
if (_.isUndefined(this.indexPattern)) {
this.indexPattern = {};
}
@ -62,7 +75,7 @@ modules.get('apps/settings')
});
_.defaults(this.indexPattern, {
id: 'filebeat-*',
id: this.patternInput.defaultValue,
title: 'filebeat-*',
fields: _(sampleFields)
.map((field, fieldName) => {

View file

@ -0,0 +1,22 @@
@import (reference) "../../../styles/_add_data_wizard";
@add-data-upload-step-multi-alert-padding: 2px;
.bulk-results {
.alert-warning {
padding: @add-data-upload-step-multi-alert-padding;
}
ul.errors {
background-color: white;
color: @text-color;
padding: @alert-padding - @add-data-upload-step-multi-alert-padding;
list-style-position: inside;
}
.alert-title {
display: flex;
padding: @alert-padding - @add-data-upload-step-multi-alert-padding;
justify-content: space-between;
}
}

View file

@ -0,0 +1,29 @@
<div ng-if="!uploadStep.created && !uploadStep.displayErrors.length">
<h2><em>Sit back, relax, we'll take it from here.</em></h2>
<div class="loading-message well">
We're loading your data now. This may take some time if you selected a large file.
</div>
</div>
<div ng-if="uploadStep.created || !!uploadStep.displayErrors.length" class="bulk-results">
<h2><em>Upload complete.</em> Let's take a look:</h2>
<div ng-if="uploadStep.created" class="alert alert-success">
Created <strong>{{ uploadStep.created }}</strong> documents!<br/>
</div>
<div class="alert alert-warning" ng-if="!!uploadStep.displayErrors.length">
<div class="alert-title">
We encountered errors while indexing your data
<a
ng-if="uploadStep.displayErrors.length > uploadStep.defaultErrorLimit"
ng-click="uploadStep.showAllErrors = !uploadStep.showAllErrors">
{{uploadStep.showAllErrors ? "Show Less" : "Show More"}}
</a>
</div>
<ul class="errors">
<li ng-repeat="error in uploadStep.displayErrors">{{ error }}</li>
</ul>
</div>
</div>

View file

@ -0,0 +1,65 @@
import modules from 'ui/modules';
import template from './upload_data_step.html';
import _ from 'lodash';
import IngestProvider from 'ui/ingest';
import './styles/_add_data_upload_data_step.less';
function formatIndexError(errorDoc) {
const lineNumber = errorDoc._id.substr(errorDoc._id.lastIndexOf(':') + 1);
const errorType = errorDoc.error.type;
const errorReason = errorDoc.error.reason;
return `Line ${lineNumber}: ${errorType} - ${errorReason}`;
}
modules.get('apps/settings')
.directive('uploadDataStep', function () {
return {
template: template,
scope: {
results: '='
},
bindToController: true,
controllerAs: 'uploadStep',
controller: function (Notifier, $window, Private, $scope) {
const ingest = Private(IngestProvider);
const notify = new Notifier({
location: 'Add Data'
});
const usePipeline = !_.isEmpty(_.get(this.results, 'pipeline.processors'));
ingest.uploadCSV(this.results.file, this.results.indexPattern.id, this.results.parseOptions.delimiter, usePipeline)
.then(
(res) => {
this.created = 0;
this.formattedErrors = [];
_.forEach(res.data, (response) => {
this.created += response.created;
this.formattedErrors = this.formattedErrors.concat(_.map(_.get(response, 'errors.index'), formatIndexError));
if (!_.isEmpty(_.get(response, 'errors.other'))) {
this.formattedErrors = this.formattedErrors.concat(response.errors.other);
}
});
},
(err) => {
notify.error(err);
$window.scrollTo(0, 0);
}
);
this.showAllErrors = false;
this.defaultErrorLimit = 10;
this.displayErrors = [];
$scope.$watchGroup(['uploadStep.formattedErrors', 'uploadStep.showAllErrors'], (newValues) => {
const [formattedErrors, showAllErrors] = newValues;
if (showAllErrors && formattedErrors) {
this.displayErrors = formattedErrors;
}
else if (formattedErrors) {
this.displayErrors = formattedErrors.slice(0, this.defaultErrorLimit + 1);
}
});
}
};
});

View file

@ -16,7 +16,7 @@ modules.get('apps/settings')
scope: {},
bindToController: true,
controllerAs: 'wizard',
controller: function ($scope, AppState, safeConfirm, kbnUrl, $http, Notifier, $window, config, Private) {
controller: function ($scope, AppState, safeConfirm, kbnUrl, Notifier, $window, Private) {
const ingest = Private(IngestProvider);
const $state = this.state = new AppState();

View file

@ -13,6 +13,13 @@
<div>
Pick this option if you already have data in Elasticsearch.
</div>
<h4>
<a href="#/settings/indices/create/upload">Upload</a>
</h4>
<div>
Got CSVs? Upload them here. No pain, all gain.
</div>
</div>
</kbn-settings-indices>
</kbn-settings-app>

View file

@ -5,6 +5,7 @@ import 'plugins/kibana/settings/sections/indices/_create';
import 'plugins/kibana/settings/sections/indices/_edit';
import 'plugins/kibana/settings/sections/indices/_field_editor';
import 'plugins/kibana/settings/sections/indices/filebeat/index';
import 'plugins/kibana/settings/sections/indices/upload/index';
import uiRoutes from 'ui/routes';
import uiModules from 'ui/modules';
import indexTemplate from 'plugins/kibana/settings/sections/indices/index.html';

View file

@ -52,6 +52,11 @@
}
}
.btn-lg {
padding: 6px 35px;
font-size: 1.2em;
}
.form-group {
margin-bottom: 5px;
}

View file

@ -0,0 +1,90 @@
<div class="wizard-container">
<div class="wizard-step-headings" ng-class="{complete: wizard.complete}">
<span
ng-class="{active: wizard.currentStep === 0}"
class="wizard-step-heading"
ng-click="wizard.setCurrentStep(0)">
1. Select
</span>
<span
ng-class="{active: wizard.currentStep === 1, aheadActive: wizard.currentStep < 1}"
class="wizard-step-heading"
ng-click="wizard.currentStep < 1 || wizard.setCurrentStep(1)">
2. Review
</span>
<span
ng-class="{active: wizard.currentStep === 2, aheadActive: wizard.currentStep < 2}"
class="wizard-step-heading"
ng-click="wizard.currentStep < 2 || wizard.setCurrentStep(2)">
3. Upload
</span>
</div>
<div ng-switch="wizard.currentStep">
<div ng-switch-when="0">
<parse-csv-step file="wizard.stepResults.file" parse-options="wizard.stepResults.parseOptions" samples="wizard.stepResults.samples"></parse-csv-step>
<div class="wizard-nav-buttons">
<div>
<button
class="btn btn-secondary"
ng-disabled="!wizard.stepResults.file"
ng-click="wizard.stepResults = undefined">
Reset
</button>
</div>
<div>
<button
class="btn btn-primary"
ng-disabled="!wizard.stepResults.samples"
ng-click="wizard.nextStep()">
Next
</button>
</div>
<div></div>
</div>
</div>
<div ng-switch-when="1">
<pattern-review-step
index-pattern="wizard.stepResults.indexPattern"
default-index-input="data"
sample-doc="wizard.stepResults.samples[0]">
</pattern-review-step>
<div class="wizard-nav-buttons">
<div>
<button
class="btn btn-secondary"
ng-click="wizard.prevStep()">
Prev
</button>
</div>
<div>
<button
class="btn btn-primary"
ng-disabled="!wizard.stepResults.indexPattern || !wizard.stepResults.indexPattern.id"
ng-click="wizard.save()">
Save
</button>
</div>
<div></div>
</div>
</div>
<div ng-switch-when="2">
<upload-data-step results="wizard.stepResults"></upload-data-step>
<div class="wizard-nav-buttons">
<div></div>
<div>
<button
class="btn btn-primary"
ng-click="wizard.nextStep()">
Done
</button>
</div>
<div></div>
</div>
</div>
</div>
</div>

View file

@ -0,0 +1,95 @@
import modules from 'ui/modules';
import template from 'plugins/kibana/settings/sections/indices/upload/directives/upload_wizard.html';
import IngestProvider from 'ui/ingest';
import 'plugins/kibana/settings/sections/indices/add_data_steps/pattern_review_step';
import 'plugins/kibana/settings/sections/indices/add_data_steps/parse_csv_step';
import 'plugins/kibana/settings/sections/indices/add_data_steps/upload_data_step';
import '../../styles/_add_data_wizard.less';
modules.get('apps/settings')
.directive('uploadWizard', function () {
return {
restrict: 'E',
template: template,
scope: {},
bindToController: true,
controllerAs: 'wizard',
controller: function ($scope, AppState, safeConfirm, kbnUrl, Notifier, $window, Private) {
const ingest = Private(IngestProvider);
const $state = this.state = new AppState();
var notify = new Notifier({
location: 'Add Data'
});
var totalSteps = 3;
this.stepResults = {};
this.setCurrentStep = (step) => {
if (!this.complete) {
$state.currentStep = step;
$state.save();
}
};
this.setCurrentStep(0);
this.nextStep = () => {
if ($state.currentStep + 1 < totalSteps) {
this.setCurrentStep($state.currentStep + 1);
}
else if ($state.currentStep + 1 === totalSteps) {
kbnUrl.change('/discover', null, {index: this.stepResults.indexPattern.id});
}
};
this.prevStep = () => {
if ($state.currentStep > 0) {
this.setCurrentStep($state.currentStep - 1);
}
};
this.save = () => {
return ingest.save(this.stepResults.indexPattern)
.then(
() => {
this.nextStep();
},
(err) => {
notify.error(err);
$window.scrollTo(0,0);
}
);
};
$scope.$watch('wizard.state.currentStep', (newValue, oldValue) => {
if (this.complete) {
$state.currentStep = totalSteps - 1;
$state.save();
return;
}
if (newValue + 1 === totalSteps) {
this.complete = true;
}
if (newValue < oldValue) {
return safeConfirm('Going back will reset any changes you\'ve made to this step, do you want to continue?')
.then(
() => {
if ($state.currentStep < 1) {
delete this.stepResults.indexPattern;
}
this.currentStep = newValue;
},
() => {
$state.currentStep = oldValue;
$state.save();
}
);
}
else {
this.currentStep = newValue;
}
});
}
};
});

View file

@ -0,0 +1,3 @@
<kbn-settings-app section="Upload a CSV">
<upload-wizard />
</kbn-settings-app>

View file

@ -0,0 +1,7 @@
import routes from 'ui/routes';
import template from 'plugins/kibana/settings/sections/indices/upload/index.html';
import './directives/upload_wizard';
routes.when('/settings/indices/create/upload', {
template: template
});

View file

@ -204,4 +204,3 @@ kbn-settings-indices {
.kbn-settings-indices-create {
.time-and-pattern > div {}
}

View file

@ -8,7 +8,8 @@ import 'ui/directives/validate_index_name';
describe('Validate index name directive', function () {
let $compile;
let $rootScope;
let html = '<input type="text" ng-model="indexName" validate-index-name />';
let noWildcardHtml = '<input type="text" ng-model="indexName" validate-index-name />';
let allowWildcardHtml = '<input type="text" ng-model="indexName" allow-wildcard validate-index-name />';
beforeEach(ngMock.module('kibana'));
@ -17,14 +18,14 @@ describe('Validate index name directive', function () {
$rootScope = _$rootScope_;
}));
function checkPattern(input) {
function checkPattern(input, html) {
$rootScope.indexName = input;
let element = $compile(html)($rootScope);
$rootScope.$digest();
return element;
}
let badPatterns = [
const badPatterns = [
null,
undefined,
'',
@ -41,19 +42,22 @@ describe('Validate index name directive', function () {
'foo,bar',
];
let goodPatterns = [
const goodPatterns = [
'...',
'foo',
'foo.bar',
'[foo-]YYYY-MM-DD',
];
const wildcardPatterns = [
'foo*',
'foo.bar*',
'foo.*',
'[foo-]YYYY-MM-DD',
'foo.*'
];
badPatterns.forEach(function (pattern) {
it('should not accept index pattern: ' + pattern, function () {
let element = checkPattern(pattern);
let element = checkPattern(pattern, noWildcardHtml);
expect(element.hasClass('ng-invalid')).to.be(true);
expect(element.hasClass('ng-valid')).to.not.be(true);
});
@ -61,7 +65,23 @@ describe('Validate index name directive', function () {
goodPatterns.forEach(function (pattern) {
it('should accept index pattern: ' + pattern, function () {
let element = checkPattern(pattern);
let element = checkPattern(pattern, noWildcardHtml);
expect(element.hasClass('ng-invalid')).to.not.be(true);
expect(element.hasClass('ng-valid')).to.be(true);
});
});
it('should disallow wildcards by default', function () {
wildcardPatterns.forEach(function (pattern) {
let element = checkPattern(pattern, noWildcardHtml);
expect(element.hasClass('ng-invalid')).to.be(true);
expect(element.hasClass('ng-valid')).to.not.be(true);
});
});
it('should allow wildcards if the allow-wildcard attribute is present', function () {
wildcardPatterns.forEach(function (pattern) {
let element = checkPattern(pattern, allowWildcardHtml);
expect(element.hasClass('ng-invalid')).to.not.be(true);
expect(element.hasClass('ng-valid')).to.be(true);
});

View file

@ -22,7 +22,7 @@ module.directive('fileUpload', function () {
const handleFile = (file) => {
if (_.isUndefined(file)) return;
if ($scope.onRead) {
if (_.has(attrs, 'onRead')) {
let reader = new FileReader();
reader.onload = function (e) {
$scope.$apply(function () {
@ -32,8 +32,10 @@ module.directive('fileUpload', function () {
reader.readAsText(file);
}
if ($scope.onLocate) {
$scope.onLocate({ file });
if (_.has(attrs, 'onLocate')) {
$scope.$apply(function () {
$scope.onLocate({ file });
});
}
};

View file

@ -8,11 +8,13 @@ uiModules
return {
restrict: 'A',
require: 'ngModel',
scope: {
'ngModel': '='
},
link: function ($scope, elem, attr, ngModel) {
let illegalCharacters = ['\\', '/', '?', '"', '<', '>', '|', ' ', ','];
const illegalCharacters = ['\\', '/', '?', '"', '<', '>', '|', ' ', ','];
const allowWildcard = !_.isUndefined(attr.allowWildcard) && attr.allowWildcard !== 'false';
if (!allowWildcard) {
illegalCharacters.push('*');
}
let isValid = function (input) {
if (input == null || input === '' || input === '.' || input === '..') return false;
@ -22,19 +24,9 @@ uiModules
return !match;
};
// From User
ngModel.$parsers.unshift(function (value) {
let valid = isValid(value);
ngModel.$setValidity('indexNameInput', valid);
return valid ? value : undefined;
});
// To user
ngModel.$formatters.unshift(function (value) {
ngModel.$setValidity('indexNameInput', isValid(value));
return value;
});
ngModel.$validators.indexNameInput = function (modelValue, viewValue) {
return isValid(viewValue);
};
}
};
});

View file

@ -0,0 +1,21 @@
import uiModules from 'ui/modules';
uiModules
.get('kibana')
.directive('validateLowercase', function () {
return {
restrict: 'A',
require: 'ngModel',
link: function ($scope, elem, attr, ctrl) {
ctrl.$validators.lowercase = function (modelValue, viewValue) {
if (ctrl.$isEmpty(modelValue)) {
// consider empty models to be valid per lowercase rules
return true;
}
return viewValue.toLowerCase() === viewValue;
};
}
};
});

View file

@ -26,7 +26,7 @@ describe('Ingest Service', function () {
it('Sets the default index if there isn\'t one already', function () {
$httpBackend
.when('POST', '../api/kibana/ingest')
.when('POST', '/api/kibana/ingest')
.respond('ok');
expect(config.get('defaultIndex')).to.be(null);
@ -38,7 +38,7 @@ describe('Ingest Service', function () {
it('Returns error from ingest API if there is one', function (done) {
$httpBackend
.expectPOST('../api/kibana/ingest')
.expectPOST('/api/kibana/ingest')
.respond(400);
ingest.save({id: 'foo'})
@ -57,7 +57,7 @@ describe('Ingest Service', function () {
it('Broadcasts an ingest:updated event on the rootScope upon succesful save', function () {
$httpBackend
.when('POST', '../api/kibana/ingest')
.when('POST', '/api/kibana/ingest')
.respond('ok');
ingest.save({id: 'foo'});
@ -75,7 +75,7 @@ describe('Ingest Service', function () {
it('Calls the DELETE endpoint of the ingest API with the given id', function () {
$httpBackend
.expectDELETE('../api/kibana/ingest/foo')
.expectDELETE('/api/kibana/ingest/foo')
.respond('ok');
ingest.delete('foo');
@ -84,7 +84,7 @@ describe('Ingest Service', function () {
it('Returns error from ingest API if there is one', function (done) {
$httpBackend
.expectDELETE('../api/kibana/ingest/foo')
.expectDELETE('/api/kibana/ingest/foo')
.respond(404);
ingest.delete('foo')
@ -103,7 +103,7 @@ describe('Ingest Service', function () {
it('Broadcasts an ingest:updated event on the rootScope upon succesful save', function () {
$httpBackend
.when('DELETE', '../api/kibana/ingest/foo')
.when('DELETE', '/api/kibana/ingest/foo')
.respond('ok');
ingest.delete('foo');
@ -114,11 +114,53 @@ describe('Ingest Service', function () {
});
});
describe('uploadCSV', function () {
it('throws an error if file and index pattern are not provided', function () {
expect(ingest.uploadCSV).to.throwException(/file is required/);
expect(ingest.uploadCSV).withArgs('foo').to.throwException(/index pattern is required/);
});
it('POSTs to the kibana _data endpoint with the correct params and the file attached as multipart/form-data', function () {
$httpBackend
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true', function (data) {
// The assertions we can do here are limited because of poor browser support for FormData methods
return data instanceof FormData;
})
.respond('ok');
const file = new Blob(['foo,bar'], {type : 'text/csv'});
ingest.uploadCSV(file, 'foo', ';', true);
$httpBackend.flush();
});
it('Returns error from the data API if there is one', function (done) {
$httpBackend
.expectPOST('/api/kibana/foo/_data?csv_delimiter=;&pipeline=true')
.respond(404);
const file = new Blob(['foo,bar'], {type : 'text/csv'});
ingest.uploadCSV(file, 'foo', ';', true)
.then(
() => {
throw new Error('expected an error response');
},
(error) => {
expect(error.status).to.be(404);
done();
}
);
$httpBackend.flush();
});
});
describe('getProcessors', () => {
it('Calls the processors GET endpoint of the ingest API', function () {
$httpBackend
.expectGET('../api/kibana/ingest/processors')
.expectGET('/api/kibana/ingest/processors')
.respond('ok');
ingest.getProcessors();
@ -127,7 +169,7 @@ describe('Ingest Service', function () {
it('Throws user-friendly error when there is an error in the request', function (done) {
$httpBackend
.when('GET', '../api/kibana/ingest/processors')
.when('GET', '/api/kibana/ingest/processors')
.respond(404);
ingest.getProcessors()

View file

@ -1,10 +1,13 @@
import PluginsKibanaSettingsSectionsIndicesRefreshKibanaIndexProvider from 'plugins/kibana/settings/sections/indices/_refresh_kibana_index';
import { keysToCamelCaseShallow, keysToSnakeCaseShallow } from '../../../plugins/kibana/common/lib/case_conversion';
import _ from 'lodash';
import angular from 'angular';
import chrome from 'ui/chrome';
export default function IngestProvider($rootScope, $http, config, $q) {
export default function IngestProvider($rootScope, $http, config, $q, Private, indexPatterns) {
const ingestAPIPrefix = '../api/kibana/ingest';
const ingestAPIPrefix = chrome.addBasePath('/api/kibana/ingest');
const refreshKibanaIndex = Private(PluginsKibanaSettingsSectionsIndicesRefreshKibanaIndexProvider);
this.save = function (indexPattern, pipeline) {
if (_.isEmpty(indexPattern)) {
@ -24,6 +27,7 @@ export default function IngestProvider($rootScope, $http, config, $q) {
config.set('defaultIndex', indexPattern.id);
}
indexPatterns.getIds.clearCache();
$rootScope.$broadcast('ingest:updated');
});
};
@ -35,6 +39,7 @@ export default function IngestProvider($rootScope, $http, config, $q) {
return $http.delete(`${ingestAPIPrefix}/${ingestId}`)
.then(() => {
indexPatterns.getIds.clearCache();
$rootScope.$broadcast('ingest:updated');
});
};
@ -71,4 +76,30 @@ export default function IngestProvider($rootScope, $http, config, $q) {
});
};
this.uploadCSV = function (file, indexPattern, delimiter, pipeline) {
if (_.isUndefined(file)) {
throw new Error('file is required');
}
if (_.isUndefined(indexPattern)) {
throw new Error('index pattern is required');
}
const formData = new FormData();
formData.append('csv', file);
const params = {};
if (!_.isUndefined(delimiter)) {
params.csv_delimiter = delimiter;
}
if (!_.isUndefined(pipeline)) {
params.pipeline = pipeline;
}
return $http.post(chrome.addBasePath(`/api/kibana/${indexPattern}/_data`), formData, {
params: params,
transformRequest: angular.identity,
headers: {'Content-Type': undefined}
});
};
}

View file

@ -138,6 +138,9 @@
@settings-filebeat-wizard-processor-container-overlay-bg: fade(#000, 10%);
// Settings - Add Data Wizard - Parse CSV
@settings-add-data-wizard-parse-csv-container-border: @kibanaBlue3;
// Visualize ===================================================================
@visualize-show-spy-border: @gray-lighter;
@visualize-show-spy-bg: @white;

View file

@ -272,6 +272,27 @@ describe('kbnUrl', function () {
expect($location.search()).to.eql({});
expect($location.hash()).to.be('');
});
it('should allow setting app state on the target url', function () {
let path = '/test/path';
let search = {search: 'test'};
let hash = 'hash';
let newPath = '/new/location';
$location.path(path).search(search).hash(hash);
// verify the starting state
expect($location.path()).to.be(path);
expect($location.search()).to.eql(search);
expect($location.hash()).to.be(hash);
kbnUrl.change(newPath, null, {foo: 'bar'});
// verify the ending state
expect($location.path()).to.be(newPath);
expect($location.search()).to.eql({_a: '(foo:bar)'});
expect($location.hash()).to.be('');
});
});
describe('changePath', function () {
@ -319,6 +340,27 @@ describe('kbnUrl', function () {
expect($location.hash()).to.be('');
});
it('should allow setting app state on the target url', function () {
let path = '/test/path';
let search = {search: 'test'};
let hash = 'hash';
let newPath = '/new/location';
$location.path(path).search(search).hash(hash);
// verify the starting state
expect($location.path()).to.be(path);
expect($location.search()).to.eql(search);
expect($location.hash()).to.be(hash);
kbnUrl.redirect(newPath, null, {foo: 'bar'});
// verify the ending state
expect($location.path()).to.be(newPath);
expect($location.search()).to.eql({_a: '(foo:bar)'});
expect($location.hash()).to.be('');
});
it('should replace the current history entry', function () {
sinon.stub($location, 'replace');
$location.url('/some/path');

View file

@ -2,6 +2,7 @@ import _ from 'lodash';
import 'ui/filters/uriescape';
import 'ui/filters/rison';
import uiModules from 'ui/modules';
import rison from 'rison-node';
uiModules.get('kibana/url')
@ -17,8 +18,8 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA
* @param {Object} [paramObj] - optional set of parameters for the url template
* @return {undefined}
*/
self.change = function (url, paramObj) {
self._changeLocation('url', url, paramObj);
self.change = function (url, paramObj, appState) {
self._changeLocation('url', url, paramObj, false, appState);
};
/**
@ -40,8 +41,8 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA
* @param {Object} [paramObj] - optional set of parameters for the url template
* @return {undefined}
*/
self.redirect = function (url, paramObj) {
self._changeLocation('url', url, paramObj, true);
self.redirect = function (url, paramObj, appState) {
self._changeLocation('url', url, paramObj, true, appState);
};
/**
@ -142,7 +143,7 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA
/////
let reloading;
self._changeLocation = function (type, url, paramObj, replace) {
self._changeLocation = function (type, url, paramObj, replace, appState) {
let prev = {
path: $location.path(),
search: $location.search()
@ -152,6 +153,10 @@ function KbnUrlProvider($route, $location, $rootScope, globalState, $parse, getA
$location[type](url);
if (replace) $location.replace();
if (appState) {
$location.search('_a', rison.encode(appState));
}
let next = {
path: $location.path(),
search: $location.search()

View file

@ -103,6 +103,16 @@ define(function (require) {
});
});
bdd.it('should use the filename and line numbers as document IDs', function () {
return request.post('/kibana/names/_data')
.attach('csv', 'test/unit/fixtures/fake_names_with_mapping_errors.csv')
.expect(200)
.then((dataResponse) => {
const id = dataResponse.body[0].errors.index[0]._id;
expect(id).to.be('fake_names_with_mapping_errors.csv:2');
});
});
bdd.it('should report any csv parsing errors under an "errors.other" key', function () {
return request.post('/kibana/names/_data')
.attach('csv', 'test/unit/fixtures/fake_names_with_parse_errors.csv')