From 984ea0700ee8b84e69f626792e1dd913607307c9 Mon Sep 17 00:00:00 2001 From: Sandra Gonzales Date: Mon, 6 Jul 2020 15:46:30 -0400 Subject: [PATCH] [Ingest Manager ] prepend kibana asset ids with package name (#70502) * prepend asset ids with package name * fix type * cleanup Co-authored-by: Elastic Machine --- .../services/epm/kibana/assets/install.ts | 119 ++++++++++++++++ .../tests/__snapshots__/install.test.ts.snap | 133 ++++++++++++++++++ .../epm/kibana/assets/tests/dashboard.json | 129 +++++++++++++++++ .../epm/kibana/assets/tests/install.test.ts | 35 +++++ .../services/epm/packages/get_objects.ts | 32 ----- .../server/services/epm/packages/index.ts | 2 +- .../server/services/epm/packages/install.ts | 58 +------- 7 files changed, 419 insertions(+), 89 deletions(-) create mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts create mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap create mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json create mode 100644 x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts delete mode 100644 x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts new file mode 100644 index 000000000000..ae6493d4716e --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/install.ts @@ -0,0 +1,119 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObject, + SavedObjectsBulkCreateObject, + SavedObjectsClientContract, +} from 'src/core/server'; +import * as Registry from '../../registry'; +import { AssetType, KibanaAssetType, AssetReference } from '../../../../types'; + +type SavedObjectToBe = Required & { type: AssetType }; +export type ArchiveAsset = Pick< + SavedObject, + 'id' | 'attributes' | 'migrationVersion' | 'references' +> & { + type: AssetType; +}; + +export async function getKibanaAsset(key: string) { + const buffer = Registry.getAsset(key); + + // cache values are buffers. convert to string / JSON + return JSON.parse(buffer.toString('utf8')); +} + +export function createSavedObjectKibanaAsset( + jsonAsset: ArchiveAsset, + pkgName: string +): SavedObjectToBe { + // convert that to an object + const asset = changeAssetIds(jsonAsset, pkgName); + + return { + type: asset.type, + id: asset.id, + attributes: asset.attributes, + references: asset.references || [], + migrationVersion: asset.migrationVersion || {}, + }; +} + +// modifies id property and the id property of references objects (not index-pattern) +// to be prepended with the package name to distinguish assets from Beats modules' assets +export const changeAssetIds = (asset: ArchiveAsset, pkgName: string): ArchiveAsset => { + const references = asset.references.map((ref) => { + if (ref.type === KibanaAssetType.indexPattern) return ref; + const id = getAssetId(ref.id, pkgName); + return { ...ref, id }; + }); + return { + ...asset, + id: getAssetId(asset.id, pkgName), + references, + }; +}; + +export const getAssetId = (id: string, pkgName: string) => { + return `${pkgName}-${id}`; +}; + +// TODO: make it an exhaustive list +// e.g. switch statement with cases for each enum key returning `never` for default case +export async function installKibanaAssets(options: { + savedObjectsClient: SavedObjectsClientContract; + pkgName: string; + paths: string[]; +}) { + const { savedObjectsClient, paths, pkgName } = options; + + // Only install Kibana assets during package installation. + const kibanaAssetTypes = Object.values(KibanaAssetType); + const installationPromises = kibanaAssetTypes.map((assetType) => + installKibanaSavedObjects({ savedObjectsClient, assetType, paths, pkgName }) + ); + + // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] + // call .flat to flatten into one dimensional array + return Promise.all(installationPromises).then((results) => results.flat()); +} + +async function installKibanaSavedObjects({ + savedObjectsClient, + assetType, + paths, + pkgName, +}: { + savedObjectsClient: SavedObjectsClientContract; + assetType: KibanaAssetType; + paths: string[]; + pkgName: string; +}) { + const isSameType = (path: string) => assetType === Registry.pathParts(path).type; + const pathsOfType = paths.filter((path) => isSameType(path)); + const kibanaAssets = await Promise.all(pathsOfType.map((path) => getKibanaAsset(path))); + const toBeSavedObjects = await Promise.all( + kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset, pkgName)) + ); + + if (toBeSavedObjects.length === 0) { + return []; + } else { + const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { + overwrite: true, + }); + const createdObjects = createResults.saved_objects; + const installed = createdObjects.map(toAssetReference); + return installed; + } +} + +function toAssetReference({ id, type }: SavedObject) { + const reference: AssetReference = { id, type: type as KibanaAssetType }; + + return reference; +} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap new file mode 100644 index 000000000000..638ed4b6118c --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap @@ -0,0 +1,133 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`a kibana asset id and its reference ids are appended with package name changeAssetIds output matches snapshot: dashboard.json 1`] = ` +{ + "attributes": { + "description": "Overview dashboard for the Nginx integration in Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "1", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 12 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 12 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 48, + "x": 0, + "y": 24 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.3.0" + } + ], + "timeRestore": false, + "title": "[Metrics Nginx] Overview ECS", + "version": 1 + }, + "id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_0", + "type": "search" + }, + { + "id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_1", + "type": "map" + }, + { + "id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_2", + "type": "dashboard" + }, + { + "id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_4", + "type": "visualization" + } + ], + "type": "dashboard" +} +`; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json new file mode 100644 index 000000000000..e28a61ae5e18 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/dashboard.json @@ -0,0 +1,129 @@ +{ + "attributes": { + "description": "Overview dashboard for the Nginx integration in Metrics", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "highlightAll": true, + "query": { + "language": "kuery", + "query": "" + }, + "version": true + } + }, + "optionsJSON": { + "darkTheme": false, + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "1", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "2", + "w": 24, + "x": 24, + "y": 12 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "3", + "w": 24, + "x": 0, + "y": 12 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "4", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "7.3.0" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 12, + "i": "5", + "w": 48, + "x": 0, + "y": 24 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "7.3.0" + } + ], + "timeRestore": false, + "title": "[Metrics Nginx] Overview ECS", + "version": 1 + }, + "id": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "metrics-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_0", + "type": "search" + }, + { + "id": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_1", + "type": "map" + }, + { + "id": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_2", + "type": "dashboard" + }, + { + "id": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs", + "name": "panel_4", + "type": "visualization" + } + ], + "type": "dashboard" +} \ No newline at end of file diff --git a/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts new file mode 100644 index 000000000000..f9bc4cdbf203 --- /dev/null +++ b/x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/install.test.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { readFileSync } from 'fs'; +import path from 'path'; +import { getAssetId, changeAssetIds } from '../install'; + +expect.addSnapshotSerializer({ + print(val) { + return JSON.stringify(val, null, 2); + }, + + test(val) { + return val; + }, +}); + +describe('a kibana asset id and its reference ids are appended with package name', () => { + const assetPath = path.join(__dirname, './dashboard.json'); + const kibanaAsset = JSON.parse(readFileSync(assetPath, 'utf-8')); + const pkgName = 'nginx'; + const modifiedAssetObject = changeAssetIds(kibanaAsset, pkgName); + + test('changeAssetIds output matches snapshot', () => { + expect(modifiedAssetObject).toMatchSnapshot(path.basename(assetPath)); + }); + + test('getAssetId', () => { + const id = '47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs'; + expect(getAssetId(id, pkgName)).toBe(`${pkgName}-${id}`); + }); +}); diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts deleted file mode 100644 index b623295c5e06..000000000000 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/get_objects.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObject, SavedObjectsBulkCreateObject } from 'src/core/server'; -import { AssetType } from '../../../types'; -import * as Registry from '../registry'; - -type ArchiveAsset = Pick; -type SavedObjectToBe = Required & { type: AssetType }; - -export async function getObject(key: string) { - const buffer = Registry.getAsset(key); - - // cache values are buffers. convert to string / JSON - const json = buffer.toString('utf8'); - // convert that to an object - const asset: ArchiveAsset = JSON.parse(json); - - const { type, file } = Registry.pathParts(key); - const savedObject: SavedObjectToBe = { - type, - id: file.replace('.json', ''), - attributes: asset.attributes, - references: asset.references || [], - migrationVersion: asset.migrationVersion || {}, - }; - - return savedObject; -} diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts index b79f9178ad6a..53ffd5c6e703 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/index.ts @@ -23,7 +23,7 @@ export { SearchParams, } from './get'; -export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install'; +export { installPackage, ensureInstalledPackage } from './install'; export { removeInstallation } from './remove'; type RequiredPackage = 'system' | 'endpoint'; diff --git a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts index 910283549abd..8f73bc9a0276 100644 --- a/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts +++ b/x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts @@ -4,13 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { SavedObject, SavedObjectsClientContract } from 'src/core/server'; +import { SavedObjectsClientContract } from 'src/core/server'; import Boom from 'boom'; import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants'; import { AssetReference, Installation, - KibanaAssetType, CallESAsCurrentUser, DefaultPackages, ElasticsearchAssetType, @@ -18,7 +17,7 @@ import { } from '../../../types'; import { installIndexPatterns } from '../kibana/index_pattern/install'; import * as Registry from '../registry'; -import { getObject } from './get_objects'; +import { installKibanaAssets } from '../kibana/assets/install'; import { getInstallation, getInstallationObject, isRequiredPackage } from './index'; import { installTemplates } from '../elasticsearch/template/install'; import { generateESIndexPatterns } from '../elasticsearch/template/template'; @@ -121,7 +120,6 @@ export async function installPackage(options: { installKibanaAssets({ savedObjectsClient, pkgName, - pkgVersion, paths, }), installPipelines(registryPackageInfo, paths, callCluster), @@ -185,27 +183,6 @@ export async function installPackage(options: { }); } -// TODO: make it an exhaustive list -// e.g. switch statement with cases for each enum key returning `never` for default case -export async function installKibanaAssets(options: { - savedObjectsClient: SavedObjectsClientContract; - pkgName: string; - pkgVersion: string; - paths: string[]; -}) { - const { savedObjectsClient, paths } = options; - - // Only install Kibana assets during package installation. - const kibanaAssetTypes = Object.values(KibanaAssetType); - const installationPromises = kibanaAssetTypes.map(async (assetType) => - installKibanaSavedObjects({ savedObjectsClient, assetType, paths }) - ); - - // installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][] - // call .flat to flatten into one dimensional array - return Promise.all(installationPromises).then((results) => results.flat()); -} - export async function saveInstallationReferences(options: { savedObjectsClient: SavedObjectsClientContract; pkgName: string; @@ -240,34 +217,3 @@ export async function saveInstallationReferences(options: { return toSaveAssetRefs; } - -async function installKibanaSavedObjects({ - savedObjectsClient, - assetType, - paths, -}: { - savedObjectsClient: SavedObjectsClientContract; - assetType: KibanaAssetType; - paths: string[]; -}) { - const isSameType = (path: string) => assetType === Registry.pathParts(path).type; - const pathsOfType = paths.filter((path) => isSameType(path)); - const toBeSavedObjects = await Promise.all(pathsOfType.map(getObject)); - - if (toBeSavedObjects.length === 0) { - return []; - } else { - const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, { - overwrite: true, - }); - const createdObjects = createResults.saved_objects; - const installed = createdObjects.map(toAssetReference); - return installed; - } -} - -function toAssetReference({ id, type }: SavedObject) { - const reference: AssetReference = { id, type: type as KibanaAssetType }; - - return reference; -}