Sample data (#17807)

* register list, install, and uninstall endpoints

* decorate server with methods needed to register data sets

* implement list endpoint, add flights sample data set

* stream data file

* create sample data index with mappings

* bulk insert into elsaticsearch

* more loadBulk work

* advance time stamp

* change http method back to post

* delete index on uninstall

* last 15 minutes example

* add option to preserver day of week and time of day

* import saved objects on install and delete saved objects on delete

* update uiSetting defaultIndex on install and uninstall

* use correct format for saved object json

* Adding example sample data, mappings and dashboards

* add sample data tab to Add Data page

* add launch button

* add sample data link to empy index pattern create state

* fix jest tests

* add toast nofication on success and fail install/uninstall

* move uiSettings of defaultIndex to client, clear index patterns get id cache

* put link to sample data sets on home page

* updated saved objects and data set

* add card for sample data

* add preview image

* updated dashboards and data set

* update button text

* woops, forgot vega

* compress data json file

* move flights data file to same folder as saved objects file

* add functional tests

* updates from chrisdavies review

* fix install API call - broken on last commit

* fix mistake in create_index_pattern

* updates from Stacey-Gammon review

* remove delete from install API

* add more tests to ensure dashboard renders as expected

* better error message on install and uninstall failure

* remove checks that may change from run to run to keep functional tests stable

* update scripted field to reflect changes in ES

* change saved object install/uninstall error code from 500 to 403

* add more logic to check if dataset is installed and display a disabled add button when there is a problem checking status

* make add data links be side-by-side on home page

* propery handle savedObjectClient bulkCreate errors. Ensure launch dashboard exists in test if dataset is installed

* ignore saved object delete 404s since users could have deleted some of the saved objects via the UI

* show response error in toast, delete index before trying to create, log create index error
This commit is contained in:
Nathan Reese 2018-05-24 14:39:07 -06:00 committed by GitHub
parent bfb002c54b
commit e1399d751f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
40 changed files with 1586 additions and 53 deletions

View file

@ -168,12 +168,16 @@ exports[`apmUiEnabled 1`] = `
/>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule
margin="l"
size="full"
/>
<EuiFlexGroup
alignItems="stretch"
component="div"
direction="row"
gutterSize="l"
justifyContent="center"
justifyContent="spaceAround"
responsive={true}
wrap={false}
>
@ -184,7 +188,37 @@ exports[`apmUiEnabled 1`] = `
<EuiText
grow={true}
>
<span
<strong
style={
Object {
"height": 38,
}
}
>
Fresh Elastic stack installation?
</strong>
<EuiLink
color="primary"
href="#/home/tutorial_directory/sampleData"
style={
Object {
"marginLeft": 8,
}
}
type="button"
>
Try some sample data sets
</EuiLink>
</EuiText>
</EuiFlexItem>
<EuiFlexItem
component="div"
grow={false}
>
<EuiText
grow={true}
>
<strong
style={
Object {
"height": 38,
@ -192,7 +226,7 @@ exports[`apmUiEnabled 1`] = `
}
>
Data already in Elasticsearch?
</span>
</strong>
<EuiLink
color="primary"
href="#/management/kibana/index"
@ -349,12 +383,16 @@ exports[`render 1`] = `
/>
</EuiFlexItem>
</EuiFlexGroup>
<EuiHorizontalRule
margin="l"
size="full"
/>
<EuiFlexGroup
alignItems="stretch"
component="div"
direction="row"
gutterSize="l"
justifyContent="center"
justifyContent="spaceAround"
responsive={true}
wrap={false}
>
@ -365,7 +403,37 @@ exports[`render 1`] = `
<EuiText
grow={true}
>
<span
<strong
style={
Object {
"height": 38,
}
}
>
Fresh Elastic stack installation?
</strong>
<EuiLink
color="primary"
href="#/home/tutorial_directory/sampleData"
style={
Object {
"marginLeft": 8,
}
}
type="button"
>
Try some sample data sets
</EuiLink>
</EuiText>
</EuiFlexItem>
<EuiFlexItem
component="div"
grow={false}
>
<EuiText
grow={true}
>
<strong
style={
Object {
"height": 38,
@ -373,7 +441,7 @@ exports[`render 1`] = `
}
>
Data already in Elasticsearch?
</span>
</strong>
<EuiLink
color="primary"
href="#/management/kibana/index"

View file

@ -13,6 +13,7 @@ import {
EuiText,
EuiCard,
EuiIcon,
EuiHorizontalRule,
} from '@elastic/eui';
export function AddData({ apmUiEnabled }) {
@ -115,12 +116,27 @@ export function AddData({ apmUiEnabled }) {
{renderCards()}
<EuiFlexGroup justifyContent="center">
<EuiHorizontalRule />
<EuiFlexGroup justifyContent="spaceAround">
<EuiFlexItem grow={false}>
<EuiText>
<span style={{ height: 38 }}>
<strong style={{ height: 38 }}>
Fresh Elastic stack installation?
</strong>
<EuiLink
style={{ marginLeft: 8 }}
href="#/home/tutorial_directory/sampleData"
>
Try some sample data sets
</EuiLink>
</EuiText>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiText>
<strong style={{ height: 38 }}>
Data already in Elasticsearch?
</span>
</strong>
<EuiLink
style={{ marginLeft: 8 }}
href="#/management/kibana/index"
@ -129,6 +145,8 @@ export function AddData({ apmUiEnabled }) {
</EuiLink>
</EuiText>
</EuiFlexItem>
</EuiFlexGroup>
</EuiPanel>

View file

@ -14,7 +14,14 @@ import { replaceTemplateStrings } from './tutorial/replace_template_strings';
import chrome from 'ui/chrome';
import { recentlyAccessedShape } from './recently_accessed';
export function HomeApp({ addBasePath, directories, recentlyAccessed }) {
export function HomeApp({
addBasePath,
directories,
recentlyAccessed,
getConfig,
setConfig,
clearIndexPatternsCache,
}) {
const isCloudEnabled = chrome.getInjected('isCloudEnabled', false);
const apmUiEnabled = chrome.getInjected('apmUiEnabled', true);
@ -25,6 +32,9 @@ export function HomeApp({ addBasePath, directories, recentlyAccessed }) {
addBasePath={addBasePath}
openTab={props.match.params.tab}
isCloudEnabled={isCloudEnabled}
getConfig={getConfig}
setConfig={setConfig}
clearIndexPatternsCache={clearIndexPatternsCache}
/>
);
};
@ -87,4 +97,7 @@ HomeApp.propTypes = {
category: PropTypes.string.isRequired
})),
recentlyAccessed: PropTypes.arrayOf(recentlyAccessedShape).isRequired,
getConfig: PropTypes.func.isRequired,
setConfig: PropTypes.func.isRequired,
clearIndexPatternsCache: PropTypes.func.isRequired,
};

View file

@ -0,0 +1,157 @@
import React from 'react';
import PropTypes from 'prop-types';
import {
EuiCard,
EuiButton,
EuiButtonEmpty,
EuiFlexGroup,
EuiFlexItem,
EuiToolTip,
} from '@elastic/eui';
import {
installSampleDataSet,
uninstallSampleDataSet
} from '../sample_data_sets';
export class SampleDataSetCard extends React.Component {
constructor(props) {
super(props);
this.state = {
isProcessingRequest: false,
};
}
startRequest = async () => {
const {
getConfig,
setConfig,
id,
name,
onRequestComplete,
defaultIndex,
clearIndexPatternsCache,
} = this.props;
this.setState({
isProcessingRequest: true,
});
if (this.isInstalled()) {
await uninstallSampleDataSet(id, name, defaultIndex, getConfig, setConfig, clearIndexPatternsCache);
} else {
await installSampleDataSet(id, name, defaultIndex, getConfig, setConfig, clearIndexPatternsCache);
}
onRequestComplete();
this.setState({
isProcessingRequest: false,
});
}
isInstalled = () => {
if (this.props.status === 'installed') {
return true;
}
return false;
}
renderBtn = () => {
switch (this.props.status) {
case 'installed':
return (
<EuiFlexGroup justifyContent="spaceBetween">
<EuiFlexItem grow={false}>
<EuiButtonEmpty
isLoading={this.state.isProcessingRequest}
onClick={this.startRequest}
color="danger"
data-test-subj={`removeSampleDataSet${this.props.id}`}
>
{this.state.isProcessingRequest ? 'Removing' : 'Remove'}
</EuiButtonEmpty>
</EuiFlexItem>
<EuiFlexItem grow={false}>
<EuiButton
href={this.props.launchUrl}
data-test-subj={`launchSampleDataSet${this.props.id}`}
>
Launch
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
);
case 'not_installed':
return (
<EuiFlexGroup justifyContent="flexEnd">
<EuiFlexItem grow={false}>
<EuiButton
isLoading={this.state.isProcessingRequest}
onClick={this.startRequest}
data-test-subj={`addSampleDataSet${this.props.id}`}
>
{this.state.isProcessingRequest ? 'Adding' : 'Add'}
</EuiButton>
</EuiFlexItem>
</EuiFlexGroup>
);
default: {
return (
<EuiFlexGroup justifyContent="flexEnd">
<EuiFlexItem grow={false}>
<EuiToolTip
position="top"
content={<p>{`Unable to verify dataset status, error: ${this.props.statusMsg}`}</p>}
>
<EuiButton
isDisabled
data-test-subj={`addSampleDataSet${this.props.id}`}
>
{'Add'}
</EuiButton>
</EuiToolTip>
</EuiFlexItem>
</EuiFlexGroup>
);
}
}
}
render() {
return (
<EuiCard
image={this.props.previewUrl}
title={this.props.name}
description={this.props.description}
betaBadgeLabel={this.isInstalled() ? 'INSTALLED' : null}
footer={this.renderBtn()}
data-test-subj={`sampleDataSetCard${this.props.id}`}
/>
);
}
}
SampleDataSetCard.propTypes = {
id: PropTypes.string.isRequired,
description: PropTypes.string.isRequired,
name: PropTypes.string.isRequired,
launchUrl: PropTypes.string.isRequired,
status: PropTypes.oneOf([
'installed',
'not_installed',
'unknown',
]).isRequired,
statusMsg: PropTypes.string,
onRequestComplete: PropTypes.func.isRequired,
getConfig: PropTypes.func.isRequired,
setConfig: PropTypes.func.isRequired,
clearIndexPatternsCache: PropTypes.func.isRequired,
defaultIndex: PropTypes.string.isRequired,
previewUrl: PropTypes.string.isRequired,
};

View file

@ -11,7 +11,7 @@ import {
EuiIcon,
} from '@elastic/eui';
export function Synopsis({ description, iconUrl, iconType, title, url, wrapInPanel }) {
export function Synopsis({ description, iconUrl, iconType, title, url, wrapInPanel, onClick }) {
let optionalImg;
if (iconUrl) {
optionalImg = (
@ -63,6 +63,18 @@ export function Synopsis({ description, iconUrl, iconType, title, url, wrapInPan
);
}
if (onClick) {
return (
<span
onClick={onClick}
className="euiLink synopsis"
data-test-subj={`homeSynopsisLink${title.toLowerCase()}`}
>
{synopsisDisplay}
</span>
);
}
return (
<a
href={url}
@ -79,5 +91,6 @@ Synopsis.propTypes = {
iconUrl: PropTypes.string,
iconType: PropTypes.string,
title: PropTypes.string.isRequired,
url: PropTypes.string.isRequired
url: PropTypes.string,
onClick: PropTypes.func,
};

View file

@ -3,6 +3,7 @@
.synopsis {
display: flex;
flex-grow: 1;
cursor: pointer;
}
.synopsis:hover {

View file

@ -2,6 +2,7 @@ import _ from 'lodash';
import React from 'react';
import PropTypes from 'prop-types';
import { Synopsis } from './synopsis';
import { SampleDataSetCard } from './sample_data_set_card';
import {
EuiPage,
@ -15,8 +16,10 @@ import {
import { getTutorials } from '../load_tutorials';
import { listSampleDataSets } from '../sample_data_sets';
const ALL = 'all';
const ALL_TAB_ID = 'all';
const SAMPLE_DATA_TAB_ID = 'sampleData';
export class TutorialDirectory extends React.Component {
@ -24,7 +27,7 @@ export class TutorialDirectory extends React.Component {
super(props);
this.tabs = [{
id: ALL,
id: ALL_TAB_ID,
name: 'All',
}, {
id: 'logging',
@ -35,30 +38,83 @@ export class TutorialDirectory extends React.Component {
}, {
id: 'security',
name: 'Security Analytics',
}, {
id: SAMPLE_DATA_TAB_ID,
name: 'Sample Data',
}];
let openTab = ALL;
let openTab = ALL_TAB_ID;
if (props.openTab && this.tabs.some(tab => { return tab.id === props.openTab; })) {
openTab = props.openTab;
}
this.state = {
selectedTabId: openTab,
tutorials: []
tutorialCards: [],
sampleDataSets: [],
};
}
async componentWillMount() {
let tutorials = await getTutorials();
componentWillUnmount() {
this._isMounted = false;
}
async componentDidMount() {
this._isMounted = true;
this.loadSampleDataSets();
const tutorialConfigs = await getTutorials();
if (!this._isMounted) {
return;
}
let tutorialCards = tutorialConfigs.map(tutorialConfig => {
return {
category: tutorialConfig.category,
icon: tutorialConfig.euiIconType,
name: tutorialConfig.name,
description: tutorialConfig.shortDescription,
url: this.props.addBasePath(`#/home/tutorial/${tutorialConfig.id}`),
elasticCloud: tutorialConfig.elasticCloud,
};
});
// Add card for sample data that only gets show in "all" tab
tutorialCards.push({
name: 'Sample Data',
description: 'Get started exploring Kibana with these "one click" data sets.',
url: this.props.addBasePath('#/home/tutorial_directory/sampleData'),
elasticCloud: true,
onClick: this.onSelectedTabChanged.bind(null, SAMPLE_DATA_TAB_ID),
});
if (this.props.isCloudEnabled) {
tutorials = tutorials.filter(tutorial => {
tutorialCards = tutorialCards.filter(tutorial => {
return _.has(tutorial, 'elasticCloud');
});
}
tutorials.sort((a, b) => {
tutorialCards.sort((a, b) => {
return a.name.toLowerCase().localeCompare(b.name.toLowerCase());
});
this.setState({ // eslint-disable-line react/no-did-mount-set-state
tutorialCards: tutorialCards,
});
}
loadSampleDataSets = async () => {
const sampleDataSets = await listSampleDataSets();
if (!this._isMounted) {
return;
}
this.setState({
tutorials: tutorials,
sampleDataSets: sampleDataSets.sort((a, b) => {
return a.name.toLowerCase().localeCompare(b.name.toLowerCase());
}),
});
}
@ -81,29 +137,58 @@ export class TutorialDirectory extends React.Component {
));
}
renderTutorials = () => {
return this.state.tutorials
renderTab = () => {
if (this.state.selectedTabId === SAMPLE_DATA_TAB_ID) {
return this.renderSampleDataSetsTab();
}
return this.renderTutorialsTab();
}
renderTutorialsTab = () => {
return this.state.tutorialCards
.filter((tutorial) => {
if (this.state.selectedTabId === ALL) {
return true;
}
return this.state.selectedTabId === tutorial.category;
return this.state.selectedTabId === ALL_TAB_ID || this.state.selectedTabId === tutorial.category;
})
.map((tutorial) => {
return (
<EuiFlexItem key={tutorial.name}>
<Synopsis
iconType={tutorial.euiIconType}
description={tutorial.shortDescription}
iconType={tutorial.icon}
description={tutorial.description}
title={tutorial.name}
wrapInPanel
url={this.props.addBasePath(`#/home/tutorial/${tutorial.id}`)}
url={tutorial.url}
onClick={tutorial.onClick}
/>
</EuiFlexItem>
);
});
};
renderSampleDataSetsTab = () => {
return this.state.sampleDataSets.map(sampleDataSet => {
return (
<EuiFlexItem key={sampleDataSet.id}>
<SampleDataSetCard
id={sampleDataSet.id}
description={sampleDataSet.description}
name={sampleDataSet.name}
launchUrl={this.props.addBasePath(`/app/kibana#/dashboard/${sampleDataSet.overviewDashboard}`)}
status={sampleDataSet.status}
statusMsg={sampleDataSet.statusMsg}
onRequestComplete={this.loadSampleDataSets}
getConfig={this.props.getConfig}
setConfig={this.props.setConfig}
clearIndexPatternsCache={this.props.clearIndexPatternsCache}
defaultIndex={sampleDataSet.defaultIndex}
previewUrl={this.props.addBasePath(sampleDataSet.previewImagePath)}
/>
</EuiFlexItem>
);
});
}
render() {
return (
<EuiPage className="home">
@ -123,7 +208,7 @@ export class TutorialDirectory extends React.Component {
</EuiTabs>
<EuiSpacer />
<EuiFlexGrid columns={4}>
{ this.renderTutorials() }
{ this.renderTab() }
</EuiFlexGrid>
</EuiPage>
@ -135,4 +220,7 @@ TutorialDirectory.propTypes = {
addBasePath: PropTypes.func.isRequired,
openTab: PropTypes.string,
isCloudEnabled: PropTypes.bool.isRequired,
getConfig: PropTypes.func.isRequired,
setConfig: PropTypes.func.isRequired,
clearIndexPatternsCache: PropTypes.func.isRequired,
};

View file

@ -2,4 +2,7 @@
add-base-path="addBasePath"
directories="directories"
recently-accessed="recentlyAccessed"
get-config="getConfig"
set-config="setConfig"
clear-index-patterns-cache="clearIndexPatternsCache"
/>

View file

@ -17,13 +17,19 @@ app.directive('homeApp', function (reactDirective) {
function getRoute() {
return {
template,
controller($scope, Private) {
controller($scope, config, indexPatterns, Private) {
$scope.addBasePath = chrome.addBasePath;
$scope.directories = Private(FeatureCatalogueRegistryProvider).inTitleOrder;
$scope.recentlyAccessed = recentlyAccessed.get().map(item => {
item.link = chrome.addBasePath(item.link);
return item;
});
$scope.getConfig = (...args) => config.get(...args);
$scope.setConfig = (...args) => config.set(...args);
$scope.clearIndexPatternsCache = () => {
const getter = indexPatterns.getIds;
getter.clearCache();
};
}
};
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 809 KiB

View file

@ -0,0 +1,96 @@
import chrome from 'ui/chrome';
import { toastNotifications } from 'ui/notify';
const sampleDataUrl = chrome.addBasePath('/api/sample_data');
const headers = new Headers({
Accept: 'application/json',
'Content-Type': 'application/json',
'kbn-xsrf': 'kibana',
});
export async function listSampleDataSets() {
try {
const response = await fetch(sampleDataUrl, {
method: 'get',
credentials: 'include',
headers: headers,
});
if (response.status >= 300) {
throw new Error(`Request failed with status code: ${response.status}`);
}
return await response.json();
} catch (err) {
toastNotifications.addDanger({
title: `Unable to load sample data sets list`,
text: `${err.message}`,
});
return [];
}
}
export async function installSampleDataSet(id, name, defaultIndex, getConfig, setConfig, clearIndexPatternsCache) {
try {
const response = await fetch(`${sampleDataUrl}/${id}`, {
method: 'post',
credentials: 'include',
headers: headers,
});
if (response.status >= 300) {
const body = await response.text();
throw new Error(`Request failed with status code: ${response.status}, message: ${body}`);
}
} catch (err) {
toastNotifications.addDanger({
title: `Unable to install sample data set: ${name}`,
text: `${err.message}`,
});
return;
}
const existingDefaultIndex = await getConfig('defaultIndex');
if (existingDefaultIndex === null) {
await setConfig('defaultIndex', defaultIndex);
}
clearIndexPatternsCache();
toastNotifications.addSuccess({
title: `${name} sample data set successfully installed`,
['data-test-subj']: 'sampleDataSetInstallToast'
});
}
export async function uninstallSampleDataSet(id, name, defaultIndex, getConfig, setConfig, clearIndexPatternsCache) {
try {
const response = await fetch(`${sampleDataUrl}/${id}`, {
method: 'delete',
credentials: 'include',
headers: headers,
});
if (response.status >= 300) {
const body = await response.text();
throw new Error(`Request failed with status code: ${response.status}, message: ${body}`);
}
} catch (err) {
toastNotifications.addDanger({
title: `Unable to uninstall sample data set`,
text: `${err.message}`,
});
return;
}
const existingDefaultIndex = await getConfig('defaultIndex');
if (existingDefaultIndex && existingDefaultIndex === defaultIndex) {
await setConfig('defaultIndex', null);
}
clearIndexPatternsCache();
toastNotifications.addSuccess({
title: `${name} sample data set successfully uninstalled`,
['data-test-subj']: 'sampleDataSetUninstallToast'
});
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 361 KiB

After

Width:  |  Height:  |  Size: 339 KiB

View file

@ -19,7 +19,9 @@ exports[`CreateIndexPatternWizard renders index pattern step when there are indi
<StepIndexPattern
allIndices={
Array [
Object {},
Object {
"name": "myIndexPattern",
},
]
}
esService={Object {}}
@ -38,7 +40,6 @@ exports[`CreateIndexPatternWizard renders the empty state when there are no indi
onChangeIncludingSystemIndices={[Function]}
/>
<EmptyState
loadingDataDocUrl=""
onRefresh={[Function]}
/>
</div>

View file

@ -69,7 +69,7 @@ describe('CreateIndexPatternWizard', () => {
component.setState({
isInitiallyLoadingIndices: false,
allIndices: [{}],
allIndices: [{ name: 'myIndexPattern' }],
});
await component.update();
@ -87,7 +87,7 @@ describe('CreateIndexPatternWizard', () => {
component.setState({
isInitiallyLoadingIndices: false,
allIndices: [{}],
allIndices: [{ name: 'myIndexPattern' }],
step: 2,
});

View file

@ -14,7 +14,6 @@ describe('CreateIndexPatternWizardRender', () => {
it('should call render', () => {
renderCreateIndexPatternWizard(
'',
'',
{
es: {},

View file

@ -51,11 +51,18 @@ exports[`EmptyState should render normally 1`] = `
 
<EuiLink
color="primary"
href="http://www.elastic.co"
target="_blank"
href="#/home/tutorial_directory"
type="button"
>
Learn how.
Learn how
</EuiLink>
or
<EuiLink
color="primary"
href="#/home/tutorial_directory/sampleData"
type="button"
>
get started with some sample data sets.
</EuiLink>
</p>
</EuiText>

View file

@ -14,7 +14,6 @@ import {
} from '@elastic/eui';
export const EmptyState = ({
loadingDataDocUrl,
onRefresh,
}) => (
<EuiPanel paddingSize="l">
@ -33,10 +32,15 @@ export const EmptyState = ({
</EuiTextColor>
&nbsp;
<EuiLink
href={loadingDataDocUrl}
target="_blank"
href="#/home/tutorial_directory"
>
Learn how.
Learn how
</EuiLink>
{' or '}
<EuiLink
href="#/home/tutorial_directory/sampleData"
>
get started with some sample data sets.
</EuiLink>
</p>
</EuiText>
@ -60,6 +64,5 @@ export const EmptyState = ({
);
EmptyState.propTypes = {
loadingDataDocUrl: PropTypes.string.isRequired,
onRefresh: PropTypes.func.isRequired,
};

View file

@ -13,7 +13,6 @@ import { getIndices } from './lib/get_indices';
export class CreateIndexPatternWizard extends Component {
static propTypes = {
loadingDataDocUrl: PropTypes.string.isRequired,
initialQuery: PropTypes.string,
services: PropTypes.shape({
es: PropTypes.object.isRequired,
@ -106,9 +105,9 @@ export class CreateIndexPatternWizard extends Component {
return <LoadingState />;
}
if (allIndices.length === 0) {
const { loadingDataDocUrl } = this.props;
return <EmptyState loadingDataDocUrl={loadingDataDocUrl} onRefresh={this.fetchIndices} />;
const hasDataIndices = allIndices.some(({ name }) => !name.startsWith('.'));
if (!hasDataIndices) {
return <EmptyState onRefresh={this.fetchIndices} />;
}
if (step === 1) {

View file

@ -2,7 +2,6 @@ import { SavedObjectsClientProvider } from 'ui/saved_objects';
import uiRoutes from 'ui/routes';
import angularTemplate from './angular_template.html';
import 'ui/index_patterns';
import { documentationLinks } from 'ui/documentation_links';
import { renderCreateIndexPatternWizard, destroyCreateIndexPatternWizard } from './render';
@ -26,7 +25,6 @@ uiRoutes.when('/management/kibana/index', {
const initialQuery = $routeParams.id ? decodeURIComponent($routeParams.id) : undefined;
renderCreateIndexPatternWizard(
documentationLinks.indexPatterns.loadingData,
initialQuery,
services
);

View file

@ -5,7 +5,6 @@ import { CreateIndexPatternWizard } from './create_index_pattern_wizard';
const CREATE_INDEX_PATTERN_DOM_ELEMENT_ID = 'createIndexPatternReact';
export function renderCreateIndexPatternWizard(
loadingDataDocUrl,
initialQuery,
services,
) {
@ -16,7 +15,6 @@ export function renderCreateIndexPatternWizard(
render(
<CreateIndexPatternWizard
loadingDataDocUrl={loadingDataDocUrl}
initialQuery={initialQuery}
services={services}
/>,

View file

@ -16,6 +16,7 @@ import optimizeMixin from '../optimize';
import * as Plugins from './plugins';
import { indexPatternsMixin } from './index_patterns';
import { savedObjectsMixin } from './saved_objects';
import { sampleDataMixin } from './sample_data';
import { kibanaIndexMappingsMixin } from './mappings';
import { serverExtensionsMixin } from './server_extensions';
import { uiMixin } from '../ui';
@ -63,6 +64,9 @@ export default class KbnServer {
// setup saved object routes
savedObjectsMixin,
// setup routes for installing/uninstalling sample data sets
sampleDataMixin,
// ensure that all bundles are built, or that the
// watch bundle server is running
optimizeMixin,

View file

@ -0,0 +1,31 @@
import Joi from 'joi';
export const dataSetSchema = {
id: Joi.string().regex(/^[a-zA-Z0-9-]+$/).required(),
name: Joi.string().required(),
description: Joi.string().required(),
previewImagePath: Joi.string().required(),
overviewDashboard: Joi.string().required(), // saved object id of main dashboard for sample data set
defaultIndex: Joi.string().required(), // saved object id of default index-pattern for sample data set
dataPath: Joi.string().required(), // path to newline delimented JSON file containing data relative to KIBANA_HOME
fields: Joi.object().required(), // Object defining Elasticsearch field mappings (contents of index.mappings.type.properties)
// times fields that will be updated relative to now when data is installed
timeFields: Joi.array().items(Joi.string()).required(),
// Reference to now in your test data set.
// When data is installed, timestamps are converted to the present time.
// The distance between a timestamp and currentTimeMarker is preserved but the date and time will change.
// For example:
// sample data set: timestamp: 2018-01-01T00:00:00Z, currentTimeMarker: 2018-01-01T12:00:00Z
// installed data set: timestamp: 2018-04-18T20:33:14Z, currentTimeMarker: 2018-04-19T08:33:14Z
currentTimeMarker: Joi.string().isoDate().required(),
// Set to true to move timestamp to current week, preserving day of week and time of day
// Relative distance from timestamp to currentTimeMarker will not remain the same
preserveDayOfWeekTimeOfDay: Joi.boolean().default(false),
// Kibana saved objects (index patter, visualizations, dashboard, ...)
// Should provide a nice demo of Kibana's functionallity with the sample data set
savedObjects: Joi.array().items(Joi.object()).required(),
};

View file

@ -0,0 +1,100 @@
import { savedObjects } from './saved_objects';
export function flightsSpecProvider() {
return {
id: 'flights',
name: 'Sample flight data',
description: 'Installs fictional flight tracking data, visualizations and dashboards to monitor plane routes.',
previewImagePath: '/plugins/kibana/home/sample_data_resources/flights/dashboard.png',
overviewDashboard: '7adfa750-4c81-11e8-b3d7-01146121b73d',
defaultIndex: 'd3d7af60-4c81-11e8-b3d7-01146121b73d',
dataPath: './src/server/sample_data/data_sets/flights/flights.json.gz',
fields: {
timestamp: {
type: 'date'
},
dayOfWeek: {
type: 'integer'
},
Carrier: {
type: 'keyword'
},
FlightNum: {
type: 'keyword'
},
Origin: {
type: 'keyword'
},
OriginAirportID: {
type: 'keyword'
},
OriginCityName: {
type: 'keyword'
},
OriginRegion: {
type: 'keyword'
},
OriginCountry: {
type: 'keyword'
},
OriginLocation: {
type: 'geo_point'
},
Dest: {
type: 'keyword'
},
DestAirportID: {
type: 'keyword'
},
DestCityName: {
type: 'keyword'
},
DestRegion: {
type: 'keyword'
},
DestCountry: {
type: 'keyword'
},
DestLocation: {
type: 'geo_point'
},
AvgTicketPrice: {
type: 'float'
},
OriginWeather: {
type: 'keyword'
},
DestWeather: {
type: 'keyword'
},
Cancelled: {
type: 'boolean'
},
DistanceMiles: {
type: 'float'
},
DistanceKilometers: {
type: 'float'
},
FlightDelayMin: {
type: 'integer'
},
FlightDelay: {
type: 'boolean'
},
FlightDelayType: {
type: 'keyword'
},
FlightTimeMin: {
type: 'float'
},
FlightTimeHour: {
type: 'keyword'
}
},
timeFields: ['timestamp'],
currentTimeMarker: '2018-01-02T00:00:00Z',
preserveDayOfWeekTimeOfDay: true,
savedObjects: savedObjects,
};
}

File diff suppressed because one or more lines are too long

View file

@ -0,0 +1 @@
export { flightsSpecProvider } from './flights';

View file

@ -0,0 +1 @@
export { sampleDataMixin } from './sample_data_mixin';

View file

@ -0,0 +1,3 @@
export { createListRoute } from './list';
export { createInstallRoute } from './install';
export { createUninstallRoute } from './uninstall';

View file

@ -0,0 +1,107 @@
import Joi from 'joi';
import { loadData } from './lib/load_data';
import { createIndexName } from './lib/create_index_name';
import { adjustTimestamp } from './lib/adjust_timestamp';
export const createInstallRoute = () => ({
path: '/api/sample_data/{id}',
method: 'POST',
config: {
validate: {
params: Joi.object().keys({
id: Joi.string().required(),
}).required()
},
handler: async (request, reply) => {
const server = request.server;
const sampleDataset = server.getSampleDatasets().find(sampleDataset => {
return sampleDataset.id === request.params.id;
});
if (!sampleDataset) {
return reply().code(404);
}
const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
const index = createIndexName(server, sampleDataset.id);
const insertCmd = {
index: {
_index: index
}
};
// clean up any old installation of dataset
try {
await callWithRequest(request, 'indices.delete', { index: index });
} catch (err) {
// ignore delete errors
}
try {
const createIndexParams = {
index: index,
body: {
settings: {
index: {
number_of_shards: 1,
number_of_replicas: 0
}
},
mappings: {
_doc: {
properties: sampleDataset.fields
}
}
}
};
await callWithRequest(request, 'indices.create', createIndexParams);
} catch (err) {
const errMsg = `Unable to create sample data index "${index}", error: ${err.message}`;
server.log(['warning'], errMsg);
return reply(errMsg).code(err.status);
}
const now = new Date();
const currentTimeMarker = new Date(Date.parse(sampleDataset.currentTimeMarker));
function updateTimestamps(doc) {
sampleDataset.timeFields.forEach(timeFieldName => {
if (doc[timeFieldName]) {
doc[timeFieldName] = adjustTimestamp(doc[timeFieldName], currentTimeMarker, now, sampleDataset.preserveDayOfWeekTimeOfDay);
}
});
return doc;
}
const bulkInsert = async (docs) => {
const bulk = [];
docs.forEach(doc => {
bulk.push(insertCmd);
bulk.push(updateTimestamps(doc));
});
const resp = await callWithRequest(request, 'bulk', { body: bulk });
if (resp.errors) {
server.log(
['warning'],
`sample_data install errors while bulk inserting. Elasticsearch response: ${JSON.stringify(resp, null, ' ')}`);
return Promise.reject(new Error(`Unable to load sample data into index "${index}", see kibana logs for details`));
}
};
loadData(sampleDataset.dataPath, bulkInsert, async (err, count) => {
if (err) {
server.log(['warning'], `sample_data install errors while loading data. Error: ${err}`);
return reply(err.message).code(500);
}
const createResults = await request.getSavedObjectsClient().bulkCreate(sampleDataset.savedObjects, { overwrite: true });
const errors = createResults.filter(savedObjectCreateResult => {
return savedObjectCreateResult.hasOwnProperty('error');
});
if (errors.length > 0) {
server.log(['warning'], `sample_data install errors while loading saved objects. Errors: ${errors.join(',')}`);
return reply(`Unable to load kibana saved objects, see kibana logs for details`).code(403);
}
return reply({ docsLoaded: count, kibanaSavedObjectsLoaded: sampleDataset.savedObjects.length });
});
}
}
});

View file

@ -0,0 +1,29 @@
const MILLISECONDS_IN_DAY = 86400000;
/**
* Convert timestamp to timestamp that is relative to now
*
* @param {String} timestamp ISO8601 formated data string YYYY-MM-dd'T'HH:mm:ss.SSS
* @param {Date} currentTimeMarker "now" reference marker in sample dataset
* @param {Date} now
* @param {Boolean} preserveDayOfWeekTimeOfDay
* @return {String} ISO8601 formated data string YYYY-MM-dd'T'HH:mm:ss.SSS of timestamp adjusted to now
*/
export function adjustTimestamp(timestamp, currentTimeMarker, now, preserveDayOfWeekTimeOfDay) {
const timestampDate = new Date(Date.parse(timestamp));
if (!preserveDayOfWeekTimeOfDay) {
// Move timestamp relative to now, preserving distance between currentTimeMarker and timestamp
const timeDelta = timestampDate.getTime() - currentTimeMarker.getTime();
return (new Date(now.getTime() + timeDelta)).toISOString();
}
// Move timestamp to current week, preserving day of week and time of day
const weekDelta = Math.round((timestampDate.getTime() - currentTimeMarker.getTime()) / (MILLISECONDS_IN_DAY * 7));
const dayOfWeekDelta = timestampDate.getUTCDay() - now.getUTCDay();
const daysDelta = dayOfWeekDelta * MILLISECONDS_IN_DAY + (weekDelta * MILLISECONDS_IN_DAY * 7);
const yearMonthDay = (new Date(now.getTime() + daysDelta)).toISOString().substring(0, 10);
return `${yearMonthDay}T${timestamp.substring(11)}`;
}

View file

@ -0,0 +1,46 @@
import { adjustTimestamp } from './adjust_timestamp';
const currentTimeMarker = new Date(Date.parse('2018-01-02T00:00:00Z'));
const now = new Date(Date.parse('2018-04-25T18:24:58.650Z')); // Wednesday
describe('relative to now', () => {
test('adjusts time to 10 minutes in past from now', () => {
const originalTimestamp = '2018-01-01T23:50:00Z'; // -10 minutes relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, false);
expect(timestamp).toBe('2018-04-25T18:14:58.650Z');
});
test('adjusts time to 1 hour in future from now', () => {
const originalTimestamp = '2018-01-02T01:00:00Z'; // + 1 hour relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, false);
expect(timestamp).toBe('2018-04-25T19:24:58.650Z');
});
});
describe('preserve day of week and time of day', () => {
test('adjusts time to monday of the same week as now', () => {
const originalTimestamp = '2018-01-01T23:50:00Z'; // Monday, same week relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, true);
expect(timestamp).toBe('2018-04-23T23:50:00Z');
});
test('adjusts time to friday of the same week as now', () => {
const originalTimestamp = '2017-12-29T23:50:00Z'; // Friday, same week relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, true);
expect(timestamp).toBe('2018-04-27T23:50:00Z');
});
test('adjusts time to monday of the previous week as now', () => {
const originalTimestamp = '2017-12-25T23:50:00Z'; // Monday, previous week relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, true);
expect(timestamp).toBe('2018-04-16T23:50:00Z');
});
test('adjusts time to friday of the week after now', () => {
const originalTimestamp = '2018-01-05T23:50:00Z'; // Friday, next week relative to currentTimeMarker
const timestamp = adjustTimestamp(originalTimestamp, currentTimeMarker, now, true);
expect(timestamp).toBe('2018-05-04T23:50:00Z');
});
});

View file

@ -0,0 +1,3 @@
export function createIndexName(server, sampleDataSetId) {
return `kibana_sample_data_${sampleDataSetId}`;
}

View file

@ -0,0 +1,78 @@
import readline from 'readline';
import fs from 'fs';
import zlib from 'zlib';
const BULK_INSERT_SIZE = 500;
export function loadData(path, bulkInsert, callback) {
let count = 0;
let docs = [];
let isPaused = false;
const readStream = fs.createReadStream(path, {
// pause does not stop lines already in buffer. Use smaller buffer size to avoid bulk inserting to many records
highWaterMark: 1024 * 4
});
const lineStream = readline.createInterface({
input: readStream.pipe(zlib.Unzip()) // eslint-disable-line new-cap
});
const onClose = async () => {
if (docs.length > 0) {
try {
await bulkInsert(docs);
} catch (err) {
callback(err);
return;
}
}
callback(null, count);
};
lineStream.on('close', onClose);
function closeWithError(err) {
lineStream.removeListener('close', onClose);
lineStream.close();
callback(err);
}
lineStream.on('line', async (line) => {
if (line.length === 0 || line.charAt(0) === '#') {
return;
}
let doc;
try {
doc = JSON.parse(line);
} catch (err) {
closeWithError(new Error(`Unable to parse line as JSON document, line: """${line}""", Error: ${err.message}`));
return;
}
count++;
docs.push(doc);
if (docs.length >= BULK_INSERT_SIZE && !isPaused) {
lineStream.pause();
// readline pause is leaky and events in buffer still get sent after pause
// need to clear buffer before async call
const docstmp = docs.slice();
docs = [];
try {
await bulkInsert(docstmp);
lineStream.resume();
} catch (err) {
closeWithError(err);
}
}
});
lineStream.on('pause', async () => {
isPaused = true;
});
lineStream.on('resume', async () => {
isPaused = false;
});
}

View file

@ -0,0 +1,13 @@
import { loadData } from './load_data';
test('load data', done => {
let myDocsCount = 0;
const bulkInsertMock = (docs) => {
myDocsCount += docs.length;
};
loadData('./src/server/sample_data/data_sets/flights/flights.json.gz', bulkInsertMock, async (err, count) => {
expect(myDocsCount).toBe(13059);
expect(count).toBe(13059);
done();
});
});

View file

@ -0,0 +1,67 @@
import _ from 'lodash';
import { createIndexName } from './lib/create_index_name';
const NOT_INSTALLED = 'not_installed';
const INSTALLED = 'installed';
const UNKNOWN = 'unknown';
export const createListRoute = () => ({
path: '/api/sample_data',
method: 'GET',
config: {
handler: async (request, reply) => {
const { callWithRequest } = request.server.plugins.elasticsearch.getCluster('data');
const sampleDatasets = request.server.getSampleDatasets().map(sampleDataset => {
return {
id: sampleDataset.id,
name: sampleDataset.name,
description: sampleDataset.description,
previewImagePath: sampleDataset.previewImagePath,
overviewDashboard: sampleDataset.overviewDashboard,
defaultIndex: sampleDataset.defaultIndex,
};
});
const isInstalledPromises = sampleDatasets.map(async sampleDataset => {
const index = createIndexName(request.server, sampleDataset.id);
try {
const indexExists = await callWithRequest(request, 'indices.exists', { index: index });
if (!indexExists) {
sampleDataset.status = NOT_INSTALLED;
return;
}
const { count } = await callWithRequest(request, 'count', { index: index });
if (count === 0) {
sampleDataset.status = NOT_INSTALLED;
return;
}
} catch (err) {
sampleDataset.status = UNKNOWN;
sampleDataset.statusMsg = err.message;
return;
}
try {
await request.getSavedObjectsClient().get('dashboard', sampleDataset.overviewDashboard);
} catch (err) {
// savedObjectClient.get() throws an boom error when object is not found.
if (_.get(err, 'output.statusCode') === 404) {
sampleDataset.status = NOT_INSTALLED;
return;
}
sampleDataset.status = UNKNOWN;
sampleDataset.statusMsg = err.message;
return;
}
sampleDataset.status = INSTALLED;
});
await Promise.all(isInstalledPromises);
reply(sampleDatasets);
}
}
});

View file

@ -0,0 +1,50 @@
import _ from 'lodash';
import Joi from 'joi';
import { createIndexName } from './lib/create_index_name';
export const createUninstallRoute = () => ({
path: '/api/sample_data/{id}',
method: 'DELETE',
config: {
validate: {
params: Joi.object().keys({
id: Joi.string().required(),
}).required()
},
handler: async (request, reply) => {
const server = request.server;
const sampleDataset = server.getSampleDatasets().find(({ id }) => {
return id === request.params.id;
});
if (!sampleDataset) {
reply().code(404);
return;
}
const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
const index = createIndexName(server, sampleDataset.id);
try {
await callWithRequest(request, 'indices.delete', { index: index });
} catch (err) {
return reply(`Unable to delete sample data index "${index}", error: ${err.message}`).code(err.status);
}
const deletePromises = sampleDataset.savedObjects.map((savedObjectJson) => {
return request.getSavedObjectsClient().delete(savedObjectJson.type, savedObjectJson.id);
});
try {
await Promise.all(deletePromises);
} catch (err) {
// ignore 404s since users could have deleted some of the saved objects via the UI
if (_.get(err, 'output.statusCode') !== 404) {
return reply(`Unable to delete samle dataset saved objects, error: ${err.message}`).code(403);
}
}
reply();
}
}
});

View file

@ -0,0 +1,50 @@
import Joi from 'joi';
import { dataSetSchema } from './data_set_schema';
import {
createListRoute,
createInstallRoute,
createUninstallRoute,
} from './routes';
import {
flightsSpecProvider,
} from './data_sets';
export function sampleDataMixin(kbnServer, server) {
server.route(createListRoute());
server.route(createInstallRoute());
server.route(createUninstallRoute());
const sampleDatasets = [];
server.decorate('server', 'getSampleDatasets', () => {
return sampleDatasets;
});
server.decorate('server', 'registerSampleDataset', (specProvider) => {
const { error, value } = Joi.validate(specProvider(server), dataSetSchema);
if (error) {
throw new Error(`Unable to register sample dataset spec because it's invalid. ${error}`);
}
const defaultIndexSavedObjectJson = value.savedObjects.find(savedObjectJson => {
return savedObjectJson.type === 'index-pattern' && savedObjectJson.id === value.defaultIndex;
});
if (!defaultIndexSavedObjectJson) {
throw new Error(
`Unable to register sample dataset spec, defaultIndex: "${value.defaultIndex}" does not exist in savedObjects list.`);
}
const dashboardSavedObjectJson = value.savedObjects.find(savedObjectJson => {
return savedObjectJson.type === 'dashboard' && savedObjectJson.id === value.overviewDashboard;
});
if (!dashboardSavedObjectJson) {
throw new Error(
`Unable to register sample dataset spec, overviewDashboard: "${value.overviewDashboard}" does not exist in savedObjects list.`);
}
sampleDatasets.push(value);
});
server.registerSampleDataset(flightsSpecProvider);
}

View file

@ -0,0 +1,92 @@
import expect from 'expect.js';
export default function ({ getService, getPageObjects }) {
const retry = getService('retry');
const find = getService('find');
const dashboardExpect = getService('dashboardExpect');
const PageObjects = getPageObjects(['common', 'header', 'home', 'dashboard']);
describe('sample data', function describeIndexTests() {
before(async () => {
await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');
await PageObjects.header.waitUntilLoadingHasFinished();
});
it('should display registered sample data sets', async ()=> {
await retry.try(async () => {
const exists = await PageObjects.home.doesSampleDataSetExist('flights');
expect(exists).to.be(true);
});
});
it('should install sample data set', async ()=> {
await PageObjects.home.addSampleDataSet('flights');
await retry.try(async () => {
const successToastExists = await PageObjects.home.doesSampleDataSetSuccessfulInstallToastExist();
expect(successToastExists).to.be(true);
});
const isInstalled = await PageObjects.home.isSampleDataSetInstalled('flights');
expect(isInstalled).to.be(true);
});
describe('dashboard', () => {
after(async () => {
await PageObjects.common.navigateToUrl('home', 'tutorial_directory/sampleData');
await PageObjects.header.waitUntilLoadingHasFinished();
});
it('should launch sample data set dashboard', async ()=> {
await PageObjects.home.launchSampleDataSet('flights');
await PageObjects.header.waitUntilLoadingHasFinished();
const today = new Date();
const todayYearMonthDay = today.toISOString().substring(0, 10);
const fromTime = `${todayYearMonthDay} 00:00:00.000`;
const toTime = `${todayYearMonthDay} 23:59:59.999`;
await PageObjects.header.setAbsoluteRange(fromTime, toTime);
const panelCount = await PageObjects.dashboard.getPanelCount();
expect(panelCount).to.be(19);
});
it('pie charts rendered', async () => {
await dashboardExpect.pieSliceCount(4);
});
it('area, bar and heatmap charts rendered', async () => {
await dashboardExpect.seriesElementCount(15);
});
it('saved searches render', async () => {
await dashboardExpect.savedSearchRowCount(50);
});
it('input controls render', async () => {
await dashboardExpect.inputControlItemCount(3);
});
it('tag cloud renders', async () => {
await dashboardExpect.tagCloudWithValuesFound(['Sunny', 'Rain', 'Clear', 'Cloudy', 'Hail']);
});
it('vega chart renders', async () => {
const tsvb = await find.existsByCssSelector('.vega-view-container');
expect(tsvb).to.be(true);
});
});
// needs to be in describe block so it is run after 'dashboard describe block'
describe('uninstall', () => {
it('should uninstall sample data set', async ()=> {
await PageObjects.home.removeSampleDataSet('flights');
await retry.try(async () => {
const successToastExists = await PageObjects.home.doesSampleDataSetSuccessfulUninstallToastExist();
expect(successToastExists).to.be(true);
});
const isInstalled = await PageObjects.home.isSampleDataSetInstalled('flights');
expect(isInstalled).to.be(false);
});
});
});
}

View file

@ -8,5 +8,6 @@ export default function ({ getService, loadTestFile }) {
loadTestFile(require.resolve('./_home'));
loadTestFile(require.resolve('./_add_data'));
loadTestFile(require.resolve('./_sample_data'));
});
}

View file

@ -16,6 +16,34 @@ export function HomePageProvider({ getService }) {
return await testSubjects.exists(`homeSynopsisLink${title}`);
}
async doesSampleDataSetExist(id) {
return await testSubjects.exists(`sampleDataSetCard${id}`);
}
async doesSampleDataSetSuccessfulInstallToastExist() {
return await testSubjects.exists('sampleDataSetInstallToast');
}
async doesSampleDataSetSuccessfulUninstallToastExist() {
return await testSubjects.exists('sampleDataSetUninstallToast');
}
async isSampleDataSetInstalled(id) {
return await testSubjects.exists(`removeSampleDataSet${id}`);
}
async addSampleDataSet(id) {
await testSubjects.click(`addSampleDataSet${id}`);
}
async removeSampleDataSet(id) {
await testSubjects.click(`removeSampleDataSet${id}`);
}
async launchSampleDataSet(id) {
await testSubjects.click(`launchSampleDataSet${id}`);
}
}
return new HomePage();