maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Ingest Manager ] prepend kibana asset ids with package name (#70502)

Browse source 
* prepend asset ids with package name

* fix type

* cleanup

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
Sandra Gonzales 2020-07-06 15:46:30 -04:00 committed by GitHub
parent b8591bc948
commit 984ea0700e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 419 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

119
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts Normal file
View file

@ -0,0 +1,119 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
SavedObject,
SavedObjectsBulkCreateObject,
SavedObjectsClientContract,
} from 'src/core/server';
import * as Registry from '../../registry';
import { AssetType, KibanaAssetType, AssetReference } from '../../../../types';
type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
export type ArchiveAsset = Pick<
SavedObject,
'id' | 'attributes' | 'migrationVersion' | 'references'
> & {
type: AssetType;
};
export async function getKibanaAsset(key: string) {
const buffer = Registry.getAsset(key);
// cache values are buffers. convert to string / JSON
return JSON.parse(buffer.toString('utf8'));
}
export function createSavedObjectKibanaAsset(
jsonAsset: ArchiveAsset,
pkgName: string
): SavedObjectToBe {
// convert that to an object
const asset = changeAssetIds(jsonAsset, pkgName);
return {
type: asset.type,
id: asset.id,
attributes: asset.attributes,
references: asset.references || [],
migrationVersion: asset.migrationVersion || {},
};
}
// modifies id property and the id property of references objects (not index-pattern)
// to be prepended with the package name to distinguish assets from Beats modules' assets
export const changeAssetIds = (asset: ArchiveAsset, pkgName: string): ArchiveAsset => {
const references = asset.references.map((ref) => {
if (ref.type === KibanaAssetType.indexPattern) return ref;
const id = getAssetId(ref.id, pkgName);
return { ...ref, id };
});
return {
...asset,
id: getAssetId(asset.id, pkgName),
references,
};
};
export const getAssetId = (id: string, pkgName: string) => {
return `${pkgName}-${id}`;
};
// TODO: make it an exhaustive list
// e.g. switch statement with cases for each enum key returning `never` for default case
export async function installKibanaAssets(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
paths: string[];
}) {
const { savedObjectsClient, paths, pkgName } = options;
// Only install Kibana assets during package installation.
const kibanaAssetTypes = Object.values(KibanaAssetType);
const installationPromises = kibanaAssetTypes.map((assetType) =>
installKibanaSavedObjects({ savedObjectsClient, assetType, paths, pkgName })
);
// installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][]
// call .flat to flatten into one dimensional array
return Promise.all(installationPromises).then((results) => results.flat());
}
async function installKibanaSavedObjects({
savedObjectsClient,
assetType,
paths,
pkgName,
}: {
savedObjectsClient: SavedObjectsClientContract;
assetType: KibanaAssetType;
paths: string[];
pkgName: string;
}) {
const isSameType = (path: string) => assetType === Registry.pathParts(path).type;
const pathsOfType = paths.filter((path) => isSameType(path));
const kibanaAssets = await Promise.all(pathsOfType.map((path) => getKibanaAsset(path)));
const toBeSavedObjects = await Promise.all(
kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset, pkgName))
);
if (toBeSavedObjects.length === 0) {
return [];
} else {
const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, {
overwrite: true,
});
const createdObjects = createResults.saved_objects;
const installed = createdObjects.map(toAssetReference);
return installed;
}
}
function toAssetReference({ id, type }: SavedObject) {
const reference: AssetReference = { id, type: type as KibanaAssetType };
return reference;
}

133
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap generated Normal file
View file

@ -0,0 +1,133 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`a kibana asset id and its reference ids are appended with package name changeAssetIds output matches snapshot: dashboard.json 1`] = `
{
"attributes": {
"description": "Overview dashboard for the Nginx integration in Metrics",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [],
"highlightAll": true,
"query": {
"language": "kuery",
"query": ""
},
"version": true
}
},
"optionsJSON": {
"darkTheme": false,
"hidePanelTitles": false,
"useMargins": true
},
"panelsJSON": [
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "1",
"w": 24,
"x": 24,
"y": 0
},
"panelIndex": "1",
"panelRefName": "panel_0",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "2",
"w": 24,
"x": 24,
"y": 12
},
"panelIndex": "2",
"panelRefName": "panel_1",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "3",
"w": 24,
"x": 0,
"y": 12
},
"panelIndex": "3",
"panelRefName": "panel_2",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "4",
"w": 24,
"x": 0,
"y": 0
},
"panelIndex": "4",
"panelRefName": "panel_3",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "5",
"w": 48,
"x": 0,
"y": 24
},
"panelIndex": "5",
"panelRefName": "panel_4",
"version": "7.3.0"
}
],
"timeRestore": false,
"title": "[Metrics Nginx] Overview ECS",
"version": 1
},
"id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs",
"migrationVersion": {
"dashboard": "7.3.0"
},
"references": [
{
"id": "metrics-*",
"name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
"type": "index-pattern"
},
{
"id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_0",
"type": "search"
},
{
"id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_1",
"type": "map"
},
{
"id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_2",
"type": "dashboard"
},
{
"id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_3",
"type": "visualization"
},
{
"id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_4",
"type": "visualization"
}
],
"type": "dashboard"
}
`;

129
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json Normal file
View file

@ -0,0 +1,129 @@
{
"attributes": {
"description": "Overview dashboard for the Nginx integration in Metrics",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": {
"filter": [],
"highlightAll": true,
"query": {
"language": "kuery",
"query": ""
},
"version": true
}
},
"optionsJSON": {
"darkTheme": false,
"hidePanelTitles": false,
"useMargins": true
},
"panelsJSON": [
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "1",
"w": 24,
"x": 24,
"y": 0
},
"panelIndex": "1",
"panelRefName": "panel_0",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "2",
"w": 24,
"x": 24,
"y": 12
},
"panelIndex": "2",
"panelRefName": "panel_1",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "3",
"w": 24,
"x": 0,
"y": 12
},
"panelIndex": "3",
"panelRefName": "panel_2",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "4",
"w": 24,
"x": 0,
"y": 0
},
"panelIndex": "4",
"panelRefName": "panel_3",
"version": "7.3.0"
},
{
"embeddableConfig": {},
"gridData": {
"h": 12,
"i": "5",
"w": 48,
"x": 0,
"y": 24
},
"panelIndex": "5",
"panelRefName": "panel_4",
"version": "7.3.0"
}
],
"timeRestore": false,
"title": "[Metrics Nginx] Overview ECS",
"version": 1
},
"id": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs",
"migrationVersion": {
"dashboard": "7.3.0"
},
"references": [
{
"id": "metrics-*",
"name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
"type": "index-pattern"
},
{
"id": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_0",
"type": "search"
},
{
"id": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_1",
"type": "map"
},
{
"id": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_2",
"type": "dashboard"
},
{
"id": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_3",
"type": "visualization"
},
{
"id": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs",
"name": "panel_4",
"type": "visualization"
}
],
"type": "dashboard"
}

35
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { readFileSync } from 'fs';
import path from 'path';
import { getAssetId, changeAssetIds } from '../install';
expect.addSnapshotSerializer({
print(val) {
return JSON.stringify(val, null, 2);
},
test(val) {
return val;
},
});
describe('a kibana asset id and its reference ids are appended with package name', () => {
const assetPath = path.join(__dirname, './dashboard.json');
const kibanaAsset = JSON.parse(readFileSync(assetPath, 'utf-8'));
const pkgName = 'nginx';
const modifiedAssetObject = changeAssetIds(kibanaAsset, pkgName);
test('changeAssetIds output matches snapshot', () => {
expect(modifiedAssetObject).toMatchSnapshot(path.basename(assetPath));
});
test('getAssetId', () => {
const id = '47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs';
expect(getAssetId(id, pkgName)).toBe(`${pkgName}-${id}`);
});
});

32
x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts
View file

@ -1,32 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObject, SavedObjectsBulkCreateObject } from 'src/core/server';
import { AssetType } from '../../../types';
import * as Registry from '../registry';
type ArchiveAsset = Pick<SavedObject, 'attributes' | 'migrationVersion' | 'references'>;
type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
export async function getObject(key: string) {
const buffer = Registry.getAsset(key);
// cache values are buffers. convert to string / JSON
const json = buffer.toString('utf8');
// convert that to an object
const asset: ArchiveAsset = JSON.parse(json);
const { type, file } = Registry.pathParts(key);
const savedObject: SavedObjectToBe = {
type,
id: file.replace('.json', ''),
attributes: asset.attributes,
references: asset.references || [],
migrationVersion: asset.migrationVersion || {},
};
return savedObject;
}

2
x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts
View file

@ -23,7 +23,7 @@ export {
SearchParams,
} from './get';
export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install';
export { installPackage, ensureInstalledPackage } from './install';
export { removeInstallation } from './remove';
type RequiredPackage = 'system' | 'endpoint';

58
x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
View file

@ -4,13 +4,12 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
import { SavedObjectsClientContract } from 'src/core/server';
import Boom from 'boom';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import {
AssetReference,
Installation,
KibanaAssetType,
CallESAsCurrentUser,
DefaultPackages,
ElasticsearchAssetType,
@ -18,7 +17,7 @@ import {
} from '../../../types';
import { installIndexPatterns } from '../kibana/index_pattern/install';
import * as Registry from '../registry';
import { getObject } from './get_objects';
import { installKibanaAssets } from '../kibana/assets/install';
import { getInstallation, getInstallationObject, isRequiredPackage } from './index';
import { installTemplates } from '../elasticsearch/template/install';
import { generateESIndexPatterns } from '../elasticsearch/template/template';
@ -121,7 +120,6 @@ export async function installPackage(options: {
installKibanaAssets({
savedObjectsClient,
pkgName,
pkgVersion,
paths,
}),
installPipelines(registryPackageInfo, paths, callCluster),
@ -185,27 +183,6 @@ export async function installPackage(options: {
});
}
// TODO: make it an exhaustive list
// e.g. switch statement with cases for each enum key returning `never` for default case
export async function installKibanaAssets(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
pkgVersion: string;
paths: string[];
}) {
const { savedObjectsClient, paths } = options;
// Only install Kibana assets during package installation.
const kibanaAssetTypes = Object.values(KibanaAssetType);
const installationPromises = kibanaAssetTypes.map(async (assetType) =>
installKibanaSavedObjects({ savedObjectsClient, assetType, paths })
);
// installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][]
// call .flat to flatten into one dimensional array
return Promise.all(installationPromises).then((results) => results.flat());
}
export async function saveInstallationReferences(options: {
savedObjectsClient: SavedObjectsClientContract;
pkgName: string;
@ -240,34 +217,3 @@ export async function saveInstallationReferences(options: {
return toSaveAssetRefs;
}
async function installKibanaSavedObjects({
savedObjectsClient,
assetType,
paths,
}: {
savedObjectsClient: SavedObjectsClientContract;
assetType: KibanaAssetType;
paths: string[];
}) {
const isSameType = (path: string) => assetType === Registry.pathParts(path).type;
const pathsOfType = paths.filter((path) => isSameType(path));
const toBeSavedObjects = await Promise.all(pathsOfType.map(getObject));
if (toBeSavedObjects.length === 0) {
return [];
} else {
const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, {
overwrite: true,
});
const createdObjects = createResults.saved_objects;
const installed = createdObjects.map(toAssetReference);
return installed;
}
}
function toAssetReference({ id, type }: SavedObject) {
const reference: AssetReference = { id, type: type as KibanaAssetType };
return reference;
}