[Ingest Manager ] prepend kibana asset ids with package name (#70502)
* prepend asset ids with package name * fix type * cleanup Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
parent
b8591bc948
commit
984ea0700e
7 changed files with 419 additions and 89 deletions
|
@ -0,0 +1,119 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import {
|
||||
SavedObject,
|
||||
SavedObjectsBulkCreateObject,
|
||||
SavedObjectsClientContract,
|
||||
} from 'src/core/server';
|
||||
import * as Registry from '../../registry';
|
||||
import { AssetType, KibanaAssetType, AssetReference } from '../../../../types';
|
||||
|
||||
type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
|
||||
export type ArchiveAsset = Pick<
|
||||
SavedObject,
|
||||
'id' | 'attributes' | 'migrationVersion' | 'references'
|
||||
> & {
|
||||
type: AssetType;
|
||||
};
|
||||
|
||||
export async function getKibanaAsset(key: string) {
|
||||
const buffer = Registry.getAsset(key);
|
||||
|
||||
// cache values are buffers. convert to string / JSON
|
||||
return JSON.parse(buffer.toString('utf8'));
|
||||
}
|
||||
|
||||
export function createSavedObjectKibanaAsset(
|
||||
jsonAsset: ArchiveAsset,
|
||||
pkgName: string
|
||||
): SavedObjectToBe {
|
||||
// convert that to an object
|
||||
const asset = changeAssetIds(jsonAsset, pkgName);
|
||||
|
||||
return {
|
||||
type: asset.type,
|
||||
id: asset.id,
|
||||
attributes: asset.attributes,
|
||||
references: asset.references || [],
|
||||
migrationVersion: asset.migrationVersion || {},
|
||||
};
|
||||
}
|
||||
|
||||
// modifies id property and the id property of references objects (not index-pattern)
|
||||
// to be prepended with the package name to distinguish assets from Beats modules' assets
|
||||
export const changeAssetIds = (asset: ArchiveAsset, pkgName: string): ArchiveAsset => {
|
||||
const references = asset.references.map((ref) => {
|
||||
if (ref.type === KibanaAssetType.indexPattern) return ref;
|
||||
const id = getAssetId(ref.id, pkgName);
|
||||
return { ...ref, id };
|
||||
});
|
||||
return {
|
||||
...asset,
|
||||
id: getAssetId(asset.id, pkgName),
|
||||
references,
|
||||
};
|
||||
};
|
||||
|
||||
export const getAssetId = (id: string, pkgName: string) => {
|
||||
return `${pkgName}-${id}`;
|
||||
};
|
||||
|
||||
// TODO: make it an exhaustive list
|
||||
// e.g. switch statement with cases for each enum key returning `never` for default case
|
||||
export async function installKibanaAssets(options: {
|
||||
savedObjectsClient: SavedObjectsClientContract;
|
||||
pkgName: string;
|
||||
paths: string[];
|
||||
}) {
|
||||
const { savedObjectsClient, paths, pkgName } = options;
|
||||
|
||||
// Only install Kibana assets during package installation.
|
||||
const kibanaAssetTypes = Object.values(KibanaAssetType);
|
||||
const installationPromises = kibanaAssetTypes.map((assetType) =>
|
||||
installKibanaSavedObjects({ savedObjectsClient, assetType, paths, pkgName })
|
||||
);
|
||||
|
||||
// installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][]
|
||||
// call .flat to flatten into one dimensional array
|
||||
return Promise.all(installationPromises).then((results) => results.flat());
|
||||
}
|
||||
|
||||
async function installKibanaSavedObjects({
|
||||
savedObjectsClient,
|
||||
assetType,
|
||||
paths,
|
||||
pkgName,
|
||||
}: {
|
||||
savedObjectsClient: SavedObjectsClientContract;
|
||||
assetType: KibanaAssetType;
|
||||
paths: string[];
|
||||
pkgName: string;
|
||||
}) {
|
||||
const isSameType = (path: string) => assetType === Registry.pathParts(path).type;
|
||||
const pathsOfType = paths.filter((path) => isSameType(path));
|
||||
const kibanaAssets = await Promise.all(pathsOfType.map((path) => getKibanaAsset(path)));
|
||||
const toBeSavedObjects = await Promise.all(
|
||||
kibanaAssets.map((asset) => createSavedObjectKibanaAsset(asset, pkgName))
|
||||
);
|
||||
|
||||
if (toBeSavedObjects.length === 0) {
|
||||
return [];
|
||||
} else {
|
||||
const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, {
|
||||
overwrite: true,
|
||||
});
|
||||
const createdObjects = createResults.saved_objects;
|
||||
const installed = createdObjects.map(toAssetReference);
|
||||
return installed;
|
||||
}
|
||||
}
|
||||
|
||||
function toAssetReference({ id, type }: SavedObject) {
|
||||
const reference: AssetReference = { id, type: type as KibanaAssetType };
|
||||
|
||||
return reference;
|
||||
}
|
133
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap
generated
Normal file
133
x-pack/plugins/ingest_manager/server/services/epm/kibana/assets/tests/__snapshots__/install.test.ts.snap
generated
Normal file
|
@ -0,0 +1,133 @@
|
|||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||
|
||||
exports[`a kibana asset id and its reference ids are appended with package name changeAssetIds output matches snapshot: dashboard.json 1`] = `
|
||||
{
|
||||
"attributes": {
|
||||
"description": "Overview dashboard for the Nginx integration in Metrics",
|
||||
"hits": 0,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": {
|
||||
"filter": [],
|
||||
"highlightAll": true,
|
||||
"query": {
|
||||
"language": "kuery",
|
||||
"query": ""
|
||||
},
|
||||
"version": true
|
||||
}
|
||||
},
|
||||
"optionsJSON": {
|
||||
"darkTheme": false,
|
||||
"hidePanelTitles": false,
|
||||
"useMargins": true
|
||||
},
|
||||
"panelsJSON": [
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "1",
|
||||
"w": 24,
|
||||
"x": 24,
|
||||
"y": 0
|
||||
},
|
||||
"panelIndex": "1",
|
||||
"panelRefName": "panel_0",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "2",
|
||||
"w": 24,
|
||||
"x": 24,
|
||||
"y": 12
|
||||
},
|
||||
"panelIndex": "2",
|
||||
"panelRefName": "panel_1",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "3",
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"panelIndex": "3",
|
||||
"panelRefName": "panel_2",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "4",
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"panelIndex": "4",
|
||||
"panelRefName": "panel_3",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "5",
|
||||
"w": 48,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"panelIndex": "5",
|
||||
"panelRefName": "panel_4",
|
||||
"version": "7.3.0"
|
||||
}
|
||||
],
|
||||
"timeRestore": false,
|
||||
"title": "[Metrics Nginx] Overview ECS",
|
||||
"version": 1
|
||||
},
|
||||
"id": "nginx-023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs",
|
||||
"migrationVersion": {
|
||||
"dashboard": "7.3.0"
|
||||
},
|
||||
"references": [
|
||||
{
|
||||
"id": "metrics-*",
|
||||
"name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
|
||||
"type": "index-pattern"
|
||||
},
|
||||
{
|
||||
"id": "nginx-555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_0",
|
||||
"type": "search"
|
||||
},
|
||||
{
|
||||
"id": "nginx-a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_1",
|
||||
"type": "map"
|
||||
},
|
||||
{
|
||||
"id": "nginx-d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_2",
|
||||
"type": "dashboard"
|
||||
},
|
||||
{
|
||||
"id": "nginx-47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_3",
|
||||
"type": "visualization"
|
||||
},
|
||||
{
|
||||
"id": "nginx-dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_4",
|
||||
"type": "visualization"
|
||||
}
|
||||
],
|
||||
"type": "dashboard"
|
||||
}
|
||||
`;
|
|
@ -0,0 +1,129 @@
|
|||
{
|
||||
"attributes": {
|
||||
"description": "Overview dashboard for the Nginx integration in Metrics",
|
||||
"hits": 0,
|
||||
"kibanaSavedObjectMeta": {
|
||||
"searchSourceJSON": {
|
||||
"filter": [],
|
||||
"highlightAll": true,
|
||||
"query": {
|
||||
"language": "kuery",
|
||||
"query": ""
|
||||
},
|
||||
"version": true
|
||||
}
|
||||
},
|
||||
"optionsJSON": {
|
||||
"darkTheme": false,
|
||||
"hidePanelTitles": false,
|
||||
"useMargins": true
|
||||
},
|
||||
"panelsJSON": [
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "1",
|
||||
"w": 24,
|
||||
"x": 24,
|
||||
"y": 0
|
||||
},
|
||||
"panelIndex": "1",
|
||||
"panelRefName": "panel_0",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "2",
|
||||
"w": 24,
|
||||
"x": 24,
|
||||
"y": 12
|
||||
},
|
||||
"panelIndex": "2",
|
||||
"panelRefName": "panel_1",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "3",
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 12
|
||||
},
|
||||
"panelIndex": "3",
|
||||
"panelRefName": "panel_2",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "4",
|
||||
"w": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
},
|
||||
"panelIndex": "4",
|
||||
"panelRefName": "panel_3",
|
||||
"version": "7.3.0"
|
||||
},
|
||||
{
|
||||
"embeddableConfig": {},
|
||||
"gridData": {
|
||||
"h": 12,
|
||||
"i": "5",
|
||||
"w": 48,
|
||||
"x": 0,
|
||||
"y": 24
|
||||
},
|
||||
"panelIndex": "5",
|
||||
"panelRefName": "panel_4",
|
||||
"version": "7.3.0"
|
||||
}
|
||||
],
|
||||
"timeRestore": false,
|
||||
"title": "[Metrics Nginx] Overview ECS",
|
||||
"version": 1
|
||||
},
|
||||
"id": "023d2930-f1a5-11e7-a9ef-93c69af7b129-ecs",
|
||||
"migrationVersion": {
|
||||
"dashboard": "7.3.0"
|
||||
},
|
||||
"references": [
|
||||
{
|
||||
"id": "metrics-*",
|
||||
"name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index",
|
||||
"type": "index-pattern"
|
||||
},
|
||||
{
|
||||
"id": "555df8a0-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_0",
|
||||
"type": "search"
|
||||
},
|
||||
{
|
||||
"id": "a1d92240-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_1",
|
||||
"type": "map"
|
||||
},
|
||||
{
|
||||
"id": "d763a570-f1a1-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_2",
|
||||
"type": "dashboard"
|
||||
},
|
||||
{
|
||||
"id": "47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_3",
|
||||
"type": "visualization"
|
||||
},
|
||||
{
|
||||
"id": "dcbffe30-f1a4-11e7-a9ef-93c69af7b129-ecs",
|
||||
"name": "panel_4",
|
||||
"type": "visualization"
|
||||
}
|
||||
],
|
||||
"type": "dashboard"
|
||||
}
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { readFileSync } from 'fs';
|
||||
import path from 'path';
|
||||
import { getAssetId, changeAssetIds } from '../install';
|
||||
|
||||
expect.addSnapshotSerializer({
|
||||
print(val) {
|
||||
return JSON.stringify(val, null, 2);
|
||||
},
|
||||
|
||||
test(val) {
|
||||
return val;
|
||||
},
|
||||
});
|
||||
|
||||
describe('a kibana asset id and its reference ids are appended with package name', () => {
|
||||
const assetPath = path.join(__dirname, './dashboard.json');
|
||||
const kibanaAsset = JSON.parse(readFileSync(assetPath, 'utf-8'));
|
||||
const pkgName = 'nginx';
|
||||
const modifiedAssetObject = changeAssetIds(kibanaAsset, pkgName);
|
||||
|
||||
test('changeAssetIds output matches snapshot', () => {
|
||||
expect(modifiedAssetObject).toMatchSnapshot(path.basename(assetPath));
|
||||
});
|
||||
|
||||
test('getAssetId', () => {
|
||||
const id = '47a8e0f0-f1a4-11e7-a9ef-93c69af7b129-ecs';
|
||||
expect(getAssetId(id, pkgName)).toBe(`${pkgName}-${id}`);
|
||||
});
|
||||
});
|
|
@ -1,32 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SavedObject, SavedObjectsBulkCreateObject } from 'src/core/server';
|
||||
import { AssetType } from '../../../types';
|
||||
import * as Registry from '../registry';
|
||||
|
||||
type ArchiveAsset = Pick<SavedObject, 'attributes' | 'migrationVersion' | 'references'>;
|
||||
type SavedObjectToBe = Required<SavedObjectsBulkCreateObject> & { type: AssetType };
|
||||
|
||||
export async function getObject(key: string) {
|
||||
const buffer = Registry.getAsset(key);
|
||||
|
||||
// cache values are buffers. convert to string / JSON
|
||||
const json = buffer.toString('utf8');
|
||||
// convert that to an object
|
||||
const asset: ArchiveAsset = JSON.parse(json);
|
||||
|
||||
const { type, file } = Registry.pathParts(key);
|
||||
const savedObject: SavedObjectToBe = {
|
||||
type,
|
||||
id: file.replace('.json', ''),
|
||||
attributes: asset.attributes,
|
||||
references: asset.references || [],
|
||||
migrationVersion: asset.migrationVersion || {},
|
||||
};
|
||||
|
||||
return savedObject;
|
||||
}
|
|
@ -23,7 +23,7 @@ export {
|
|||
SearchParams,
|
||||
} from './get';
|
||||
|
||||
export { installKibanaAssets, installPackage, ensureInstalledPackage } from './install';
|
||||
export { installPackage, ensureInstalledPackage } from './install';
|
||||
export { removeInstallation } from './remove';
|
||||
|
||||
type RequiredPackage = 'system' | 'endpoint';
|
||||
|
|
|
@ -4,13 +4,12 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { SavedObject, SavedObjectsClientContract } from 'src/core/server';
|
||||
import { SavedObjectsClientContract } from 'src/core/server';
|
||||
import Boom from 'boom';
|
||||
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
|
||||
import {
|
||||
AssetReference,
|
||||
Installation,
|
||||
KibanaAssetType,
|
||||
CallESAsCurrentUser,
|
||||
DefaultPackages,
|
||||
ElasticsearchAssetType,
|
||||
|
@ -18,7 +17,7 @@ import {
|
|||
} from '../../../types';
|
||||
import { installIndexPatterns } from '../kibana/index_pattern/install';
|
||||
import * as Registry from '../registry';
|
||||
import { getObject } from './get_objects';
|
||||
import { installKibanaAssets } from '../kibana/assets/install';
|
||||
import { getInstallation, getInstallationObject, isRequiredPackage } from './index';
|
||||
import { installTemplates } from '../elasticsearch/template/install';
|
||||
import { generateESIndexPatterns } from '../elasticsearch/template/template';
|
||||
|
@ -121,7 +120,6 @@ export async function installPackage(options: {
|
|||
installKibanaAssets({
|
||||
savedObjectsClient,
|
||||
pkgName,
|
||||
pkgVersion,
|
||||
paths,
|
||||
}),
|
||||
installPipelines(registryPackageInfo, paths, callCluster),
|
||||
|
@ -185,27 +183,6 @@ export async function installPackage(options: {
|
|||
});
|
||||
}
|
||||
|
||||
// TODO: make it an exhaustive list
|
||||
// e.g. switch statement with cases for each enum key returning `never` for default case
|
||||
export async function installKibanaAssets(options: {
|
||||
savedObjectsClient: SavedObjectsClientContract;
|
||||
pkgName: string;
|
||||
pkgVersion: string;
|
||||
paths: string[];
|
||||
}) {
|
||||
const { savedObjectsClient, paths } = options;
|
||||
|
||||
// Only install Kibana assets during package installation.
|
||||
const kibanaAssetTypes = Object.values(KibanaAssetType);
|
||||
const installationPromises = kibanaAssetTypes.map(async (assetType) =>
|
||||
installKibanaSavedObjects({ savedObjectsClient, assetType, paths })
|
||||
);
|
||||
|
||||
// installKibanaSavedObjects returns AssetReference[], so .map creates AssetReference[][]
|
||||
// call .flat to flatten into one dimensional array
|
||||
return Promise.all(installationPromises).then((results) => results.flat());
|
||||
}
|
||||
|
||||
export async function saveInstallationReferences(options: {
|
||||
savedObjectsClient: SavedObjectsClientContract;
|
||||
pkgName: string;
|
||||
|
@ -240,34 +217,3 @@ export async function saveInstallationReferences(options: {
|
|||
|
||||
return toSaveAssetRefs;
|
||||
}
|
||||
|
||||
async function installKibanaSavedObjects({
|
||||
savedObjectsClient,
|
||||
assetType,
|
||||
paths,
|
||||
}: {
|
||||
savedObjectsClient: SavedObjectsClientContract;
|
||||
assetType: KibanaAssetType;
|
||||
paths: string[];
|
||||
}) {
|
||||
const isSameType = (path: string) => assetType === Registry.pathParts(path).type;
|
||||
const pathsOfType = paths.filter((path) => isSameType(path));
|
||||
const toBeSavedObjects = await Promise.all(pathsOfType.map(getObject));
|
||||
|
||||
if (toBeSavedObjects.length === 0) {
|
||||
return [];
|
||||
} else {
|
||||
const createResults = await savedObjectsClient.bulkCreate(toBeSavedObjects, {
|
||||
overwrite: true,
|
||||
});
|
||||
const createdObjects = createResults.saved_objects;
|
||||
const installed = createdObjects.map(toAssetReference);
|
||||
return installed;
|
||||
}
|
||||
}
|
||||
|
||||
function toAssetReference({ id, type }: SavedObject) {
|
||||
const reference: AssetReference = { id, type: type as KibanaAssetType };
|
||||
|
||||
return reference;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue