[SecuritySolution-Ingest]: use new metadata current and add transform installation (#74394)
[SecuritySolution-Ingest]: use new metadata current and add transform installation
This commit is contained in:
parent
194d0b0a3f
commit
2ed4b57776
21 changed files with 965 additions and 186 deletions
|
@ -40,6 +40,7 @@ export enum ElasticsearchAssetType {
|
||||||
ingestPipeline = 'ingest_pipeline',
|
ingestPipeline = 'ingest_pipeline',
|
||||||
indexTemplate = 'index_template',
|
indexTemplate = 'index_template',
|
||||||
ilmPolicy = 'ilm_policy',
|
ilmPolicy = 'ilm_policy',
|
||||||
|
transform = 'transform',
|
||||||
}
|
}
|
||||||
|
|
||||||
export enum AgentAssetType {
|
export enum AgentAssetType {
|
||||||
|
|
|
@ -19,6 +19,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
|
||||||
dashboard: 'Dashboard',
|
dashboard: 'Dashboard',
|
||||||
ilm_policy: 'ILM Policy',
|
ilm_policy: 'ILM Policy',
|
||||||
ingest_pipeline: 'Ingest Pipeline',
|
ingest_pipeline: 'Ingest Pipeline',
|
||||||
|
transform: 'Transform',
|
||||||
'index-pattern': 'Index Pattern',
|
'index-pattern': 'Index Pattern',
|
||||||
index_template: 'Index Template',
|
index_template: 'Index Template',
|
||||||
component_template: 'Component Template',
|
component_template: 'Component Template',
|
||||||
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import * as Registry from '../../registry';
|
||||||
|
|
||||||
|
export const getAsset = (path: string): Buffer => {
|
||||||
|
return Registry.getAsset(path);
|
||||||
|
};
|
|
@ -0,0 +1,165 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { SavedObjectsClientContract } from 'kibana/server';
|
||||||
|
|
||||||
|
import { saveInstalledEsRefs } from '../../packages/install';
|
||||||
|
import * as Registry from '../../registry';
|
||||||
|
import {
|
||||||
|
Dataset,
|
||||||
|
ElasticsearchAssetType,
|
||||||
|
EsAssetReference,
|
||||||
|
RegistryPackage,
|
||||||
|
} from '../../../../../common/types/models';
|
||||||
|
import { CallESAsCurrentUser } from '../../../../types';
|
||||||
|
import { getInstallation } from '../../packages';
|
||||||
|
import { deleteTransforms, deleteTransformRefs } from './remove';
|
||||||
|
import { getAsset } from './common';
|
||||||
|
|
||||||
|
interface TransformInstallation {
|
||||||
|
installationName: string;
|
||||||
|
content: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TransformPathDataset {
|
||||||
|
path: string;
|
||||||
|
dataset: Dataset;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const installTransformForDataset = async (
|
||||||
|
registryPackage: RegistryPackage,
|
||||||
|
paths: string[],
|
||||||
|
callCluster: CallESAsCurrentUser,
|
||||||
|
savedObjectsClient: SavedObjectsClientContract
|
||||||
|
) => {
|
||||||
|
const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name });
|
||||||
|
let previousInstalledTransformEsAssets: EsAssetReference[] = [];
|
||||||
|
if (installation) {
|
||||||
|
previousInstalledTransformEsAssets = installation.installed_es.filter(
|
||||||
|
({ type, id }) => type === ElasticsearchAssetType.transform
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete all previous transform
|
||||||
|
await deleteTransforms(
|
||||||
|
callCluster,
|
||||||
|
previousInstalledTransformEsAssets.map((asset) => asset.id)
|
||||||
|
);
|
||||||
|
// install the latest dataset
|
||||||
|
const datasets = registryPackage.datasets;
|
||||||
|
if (!datasets?.length) return [];
|
||||||
|
const installNameSuffix = `${registryPackage.version}`;
|
||||||
|
|
||||||
|
const transformPaths = paths.filter((path) => isTransform(path));
|
||||||
|
let installedTransforms: EsAssetReference[] = [];
|
||||||
|
if (transformPaths.length > 0) {
|
||||||
|
const transformPathDatasets = datasets.reduce<TransformPathDataset[]>((acc, dataset) => {
|
||||||
|
transformPaths.forEach((path) => {
|
||||||
|
if (isDatasetTransform(path, dataset.path)) {
|
||||||
|
acc.push({ path, dataset });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
return acc;
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const transformRefs = transformPathDatasets.reduce<EsAssetReference[]>(
|
||||||
|
(acc, transformPathDataset) => {
|
||||||
|
if (transformPathDataset) {
|
||||||
|
acc.push({
|
||||||
|
id: getTransformNameForInstallation(transformPathDataset, installNameSuffix),
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
[]
|
||||||
|
);
|
||||||
|
|
||||||
|
// get and save transform refs before installing transforms
|
||||||
|
await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, transformRefs);
|
||||||
|
|
||||||
|
const transforms: TransformInstallation[] = transformPathDatasets.map(
|
||||||
|
(transformPathDataset: TransformPathDataset) => {
|
||||||
|
return {
|
||||||
|
installationName: getTransformNameForInstallation(
|
||||||
|
transformPathDataset,
|
||||||
|
installNameSuffix
|
||||||
|
),
|
||||||
|
content: getAsset(transformPathDataset.path).toString('utf-8'),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
const installationPromises = transforms.map(async (transform) => {
|
||||||
|
return installTransform({ callCluster, transform });
|
||||||
|
});
|
||||||
|
|
||||||
|
installedTransforms = await Promise.all(installationPromises).then((results) => results.flat());
|
||||||
|
}
|
||||||
|
|
||||||
|
if (previousInstalledTransformEsAssets.length > 0) {
|
||||||
|
const currentInstallation = await getInstallation({
|
||||||
|
savedObjectsClient,
|
||||||
|
pkgName: registryPackage.name,
|
||||||
|
});
|
||||||
|
|
||||||
|
// remove the saved object reference
|
||||||
|
await deleteTransformRefs(
|
||||||
|
savedObjectsClient,
|
||||||
|
currentInstallation?.installed_es || [],
|
||||||
|
registryPackage.name,
|
||||||
|
previousInstalledTransformEsAssets.map((asset) => asset.id),
|
||||||
|
installedTransforms.map((installed) => installed.id)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return installedTransforms;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isTransform = (path: string) => {
|
||||||
|
const pathParts = Registry.pathParts(path);
|
||||||
|
return pathParts.type === ElasticsearchAssetType.transform;
|
||||||
|
};
|
||||||
|
|
||||||
|
const isDatasetTransform = (path: string, datasetName: string) => {
|
||||||
|
const pathParts = Registry.pathParts(path);
|
||||||
|
return (
|
||||||
|
!path.endsWith('/') &&
|
||||||
|
pathParts.type === ElasticsearchAssetType.transform &&
|
||||||
|
pathParts.dataset !== undefined &&
|
||||||
|
datasetName === pathParts.dataset
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
async function installTransform({
|
||||||
|
callCluster,
|
||||||
|
transform,
|
||||||
|
}: {
|
||||||
|
callCluster: CallESAsCurrentUser;
|
||||||
|
transform: TransformInstallation;
|
||||||
|
}): Promise<EsAssetReference> {
|
||||||
|
// defer validation on put if the source index is not available
|
||||||
|
await callCluster('transport.request', {
|
||||||
|
method: 'PUT',
|
||||||
|
path: `_transform/${transform.installationName}`,
|
||||||
|
query: 'defer_validation=true',
|
||||||
|
body: transform.content,
|
||||||
|
});
|
||||||
|
|
||||||
|
await callCluster('transport.request', {
|
||||||
|
method: 'POST',
|
||||||
|
path: `_transform/${transform.installationName}/_start`,
|
||||||
|
});
|
||||||
|
|
||||||
|
return { id: transform.installationName, type: ElasticsearchAssetType.transform };
|
||||||
|
}
|
||||||
|
|
||||||
|
const getTransformNameForInstallation = (
|
||||||
|
transformDataset: TransformPathDataset,
|
||||||
|
suffix: string
|
||||||
|
) => {
|
||||||
|
const filename = transformDataset?.path.split('/')?.pop()?.split('.')[0];
|
||||||
|
return `${transformDataset.dataset.type}-${transformDataset.dataset.name}-${filename}-${suffix}`;
|
||||||
|
};
|
|
@ -0,0 +1,69 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { SavedObjectsClientContract } from 'kibana/server';
|
||||||
|
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||||
|
import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
|
||||||
|
import { deleteTransformRefs } from './remove';
|
||||||
|
import { EsAssetReference } from '../../../../../common/types/models';
|
||||||
|
|
||||||
|
describe('test transform install', () => {
|
||||||
|
let savedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
|
||||||
|
beforeEach(() => {
|
||||||
|
savedObjectsClient = savedObjectsClientMock.create();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('can delete transform ref and handle duplicate when previous version and current version are the same', async () => {
|
||||||
|
await deleteTransformRefs(
|
||||||
|
savedObjectsClient,
|
||||||
|
[
|
||||||
|
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
|
||||||
|
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
|
||||||
|
] as EsAssetReference[],
|
||||||
|
'endpoint',
|
||||||
|
['metrics-endpoint.metadata-current-default-0.16.0-dev.0'],
|
||||||
|
['metrics-endpoint.metadata-current-default-0.16.0-dev.0']
|
||||||
|
);
|
||||||
|
expect(savedObjectsClient.update.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [
|
||||||
|
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
|
||||||
|
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('can delete transform ref when previous version and current version are not the same', async () => {
|
||||||
|
await deleteTransformRefs(
|
||||||
|
savedObjectsClient,
|
||||||
|
[
|
||||||
|
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
|
||||||
|
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
|
||||||
|
] as EsAssetReference[],
|
||||||
|
'endpoint',
|
||||||
|
['metrics-endpoint.metadata-current-default-0.15.0-dev.0'],
|
||||||
|
['metrics-endpoint.metadata-current-default-0.16.0-dev.0']
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(savedObjectsClient.update.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [
|
||||||
|
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
|
||||||
|
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
|
@ -0,0 +1,58 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { SavedObjectsClientContract } from 'kibana/server';
|
||||||
|
import { CallESAsCurrentUser, ElasticsearchAssetType, EsAssetReference } from '../../../../types';
|
||||||
|
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants';
|
||||||
|
|
||||||
|
export const stopTransforms = async (transformIds: string[], callCluster: CallESAsCurrentUser) => {
|
||||||
|
for (const transformId of transformIds) {
|
||||||
|
await callCluster('transport.request', {
|
||||||
|
method: 'POST',
|
||||||
|
path: `_transform/${transformId}/_stop`,
|
||||||
|
query: 'force=true',
|
||||||
|
ignore: [404],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deleteTransforms = async (
|
||||||
|
callCluster: CallESAsCurrentUser,
|
||||||
|
transformIds: string[]
|
||||||
|
) => {
|
||||||
|
await Promise.all(
|
||||||
|
transformIds.map(async (transformId) => {
|
||||||
|
await stopTransforms([transformId], callCluster);
|
||||||
|
await callCluster('transport.request', {
|
||||||
|
method: 'DELETE',
|
||||||
|
query: 'force=true',
|
||||||
|
path: `_transform/${transformId}`,
|
||||||
|
ignore: [404],
|
||||||
|
});
|
||||||
|
})
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
|
export const deleteTransformRefs = async (
|
||||||
|
savedObjectsClient: SavedObjectsClientContract,
|
||||||
|
installedEsAssets: EsAssetReference[],
|
||||||
|
pkgName: string,
|
||||||
|
installedEsIdToRemove: string[],
|
||||||
|
currentInstalledEsTransformIds: string[]
|
||||||
|
) => {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const filteredAssets = installedEsAssets.filter(({ type, id }) => {
|
||||||
|
if (type !== ElasticsearchAssetType.transform) return true;
|
||||||
|
const add =
|
||||||
|
(currentInstalledEsTransformIds.includes(id) || !installedEsIdToRemove.includes(id)) &&
|
||||||
|
!seen.has(id);
|
||||||
|
seen.add(id);
|
||||||
|
return add;
|
||||||
|
});
|
||||||
|
return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
|
||||||
|
installed_es: filteredAssets,
|
||||||
|
});
|
||||||
|
};
|
|
@ -0,0 +1,420 @@
|
||||||
|
/*
|
||||||
|
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||||
|
* or more contributor license agreements. Licensed under the Elastic License;
|
||||||
|
* you may not use this file except in compliance with the Elastic License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
jest.mock('../../packages/get', () => {
|
||||||
|
return { getInstallation: jest.fn(), getInstallationObject: jest.fn() };
|
||||||
|
});
|
||||||
|
|
||||||
|
jest.mock('./common', () => {
|
||||||
|
return {
|
||||||
|
getAsset: jest.fn(),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
import { installTransformForDataset } from './install';
|
||||||
|
import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server';
|
||||||
|
import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types';
|
||||||
|
import { getInstallation, getInstallationObject } from '../../packages';
|
||||||
|
import { getAsset } from './common';
|
||||||
|
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
|
||||||
|
import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
|
||||||
|
|
||||||
|
describe('test transform install', () => {
|
||||||
|
let legacyScopedClusterClient: jest.Mocked<ILegacyScopedClusterClient>;
|
||||||
|
let savedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
|
||||||
|
beforeEach(() => {
|
||||||
|
legacyScopedClusterClient = {
|
||||||
|
callAsInternalUser: jest.fn(),
|
||||||
|
callAsCurrentUser: jest.fn(),
|
||||||
|
};
|
||||||
|
(getInstallation as jest.MockedFunction<typeof getInstallation>).mockReset();
|
||||||
|
(getInstallationObject as jest.MockedFunction<typeof getInstallationObject>).mockReset();
|
||||||
|
savedObjectsClient = savedObjectsClientMock.create();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
jest.clearAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
test('can install new versions and removes older version', async () => {
|
||||||
|
const previousInstallation: Installation = ({
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.policy-0.16.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.ingestPipeline,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
|
||||||
|
const currentInstallation: Installation = ({
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.policy-0.16.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.ingestPipeline,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
(getAsset as jest.MockedFunction<typeof getAsset>)
|
||||||
|
.mockReturnValueOnce(Buffer.from('{"content": "data"}', 'utf8'))
|
||||||
|
.mockReturnValueOnce(Buffer.from('{"content": "data"}', 'utf8'));
|
||||||
|
|
||||||
|
(getInstallation as jest.MockedFunction<typeof getInstallation>)
|
||||||
|
.mockReturnValueOnce(Promise.resolve(previousInstallation))
|
||||||
|
.mockReturnValueOnce(Promise.resolve(currentInstallation));
|
||||||
|
|
||||||
|
(getInstallationObject as jest.MockedFunction<
|
||||||
|
typeof getInstallationObject
|
||||||
|
>).mockReturnValueOnce(
|
||||||
|
Promise.resolve(({
|
||||||
|
attributes: {
|
||||||
|
installed_es: previousInstallation.installed_es,
|
||||||
|
},
|
||||||
|
} as unknown) as SavedObject<Installation>)
|
||||||
|
);
|
||||||
|
|
||||||
|
await installTransformForDataset(
|
||||||
|
({
|
||||||
|
name: 'endpoint',
|
||||||
|
version: '0.16.0-dev.0',
|
||||||
|
datasets: [
|
||||||
|
{
|
||||||
|
type: 'metrics',
|
||||||
|
name: 'endpoint.metadata',
|
||||||
|
title: 'Endpoint Metadata',
|
||||||
|
release: 'experimental',
|
||||||
|
package: 'endpoint',
|
||||||
|
ingest_pipeline: 'default',
|
||||||
|
elasticsearch: {
|
||||||
|
'index_template.mappings': {
|
||||||
|
dynamic: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
path: 'metadata',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'metrics',
|
||||||
|
name: 'endpoint.metadata_current',
|
||||||
|
title: 'Endpoint Metadata Current',
|
||||||
|
release: 'experimental',
|
||||||
|
package: 'endpoint',
|
||||||
|
ingest_pipeline: 'default',
|
||||||
|
elasticsearch: {
|
||||||
|
'index_template.mappings': {
|
||||||
|
dynamic: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
path: 'metadata_current',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as RegistryPackage,
|
||||||
|
[
|
||||||
|
'endpoint-0.16.0-dev.0/dataset/policy/elasticsearch/ingest_pipeline/default.json',
|
||||||
|
'endpoint-0.16.0-dev.0/dataset/metadata/elasticsearch/transform/default.json',
|
||||||
|
'endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json',
|
||||||
|
],
|
||||||
|
legacyScopedClusterClient.callAsCurrentUser,
|
||||||
|
savedObjectsClient
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0/_stop',
|
||||||
|
query: 'force=true',
|
||||||
|
ignore: [404],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'DELETE',
|
||||||
|
query: 'force=true',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0',
|
||||||
|
ignore: [404],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'PUT',
|
||||||
|
path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0',
|
||||||
|
query: 'defer_validation=true',
|
||||||
|
body: '{"content": "data"}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'PUT',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
|
||||||
|
query: 'defer_validation=true',
|
||||||
|
body: '{"content": "data"}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0/_start',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
|
||||||
|
expect(savedObjectsClient.update.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.policy-0.16.0-dev.0',
|
||||||
|
type: 'ingest_pipeline',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.policy-0.16.0-dev.0',
|
||||||
|
type: 'ingest_pipeline',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('can install new version and when no older version', async () => {
|
||||||
|
const previousInstallation: Installation = ({
|
||||||
|
installed_es: [],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
|
||||||
|
const currentInstallation: Installation = ({
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
(getAsset as jest.MockedFunction<typeof getAsset>).mockReturnValueOnce(
|
||||||
|
Buffer.from('{"content": "data"}', 'utf8')
|
||||||
|
);
|
||||||
|
(getInstallation as jest.MockedFunction<typeof getInstallation>)
|
||||||
|
.mockReturnValueOnce(Promise.resolve(previousInstallation))
|
||||||
|
.mockReturnValueOnce(Promise.resolve(currentInstallation));
|
||||||
|
|
||||||
|
(getInstallationObject as jest.MockedFunction<
|
||||||
|
typeof getInstallationObject
|
||||||
|
>).mockReturnValueOnce(
|
||||||
|
Promise.resolve(({ attributes: { installed_es: [] } } as unknown) as SavedObject<
|
||||||
|
Installation
|
||||||
|
>)
|
||||||
|
);
|
||||||
|
legacyScopedClusterClient.callAsCurrentUser = jest.fn();
|
||||||
|
await installTransformForDataset(
|
||||||
|
({
|
||||||
|
name: 'endpoint',
|
||||||
|
version: '0.16.0-dev.0',
|
||||||
|
datasets: [
|
||||||
|
{
|
||||||
|
type: 'metrics',
|
||||||
|
name: 'endpoint.metadata_current',
|
||||||
|
title: 'Endpoint Metadata',
|
||||||
|
release: 'experimental',
|
||||||
|
package: 'endpoint',
|
||||||
|
ingest_pipeline: 'default',
|
||||||
|
elasticsearch: {
|
||||||
|
'index_template.mappings': {
|
||||||
|
dynamic: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
path: 'metadata_current',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as RegistryPackage,
|
||||||
|
['endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json'],
|
||||||
|
legacyScopedClusterClient.callAsCurrentUser,
|
||||||
|
savedObjectsClient
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'PUT',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
|
||||||
|
query: 'defer_validation=true',
|
||||||
|
body: '{"content": "data"}',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
method: 'POST',
|
||||||
|
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
expect(savedObjectsClient.update.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [
|
||||||
|
{ id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('can removes older version when no new install in package', async () => {
|
||||||
|
const previousInstallation: Installation = ({
|
||||||
|
installed_es: [
|
||||||
|
{
|
||||||
|
id: 'metrics-endpoint.metadata-current-default-0.15.0-dev.0',
|
||||||
|
type: ElasticsearchAssetType.transform,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
|
||||||
|
const currentInstallation: Installation = ({
|
||||||
|
installed_es: [],
|
||||||
|
} as unknown) as Installation;
|
||||||
|
|
||||||
|
(getInstallation as jest.MockedFunction<typeof getInstallation>)
|
||||||
|
.mockReturnValueOnce(Promise.resolve(previousInstallation))
|
||||||
|
.mockReturnValueOnce(Promise.resolve(currentInstallation));
|
||||||
|
|
||||||
|
(getInstallationObject as jest.MockedFunction<
|
||||||
|
typeof getInstallationObject
|
||||||
|
>).mockReturnValueOnce(
|
||||||
|
Promise.resolve(({
|
||||||
|
attributes: { installed_es: currentInstallation.installed_es },
|
||||||
|
} as unknown) as SavedObject<Installation>)
|
||||||
|
);
|
||||||
|
|
||||||
|
await installTransformForDataset(
|
||||||
|
({
|
||||||
|
name: 'endpoint',
|
||||||
|
version: '0.16.0-dev.0',
|
||||||
|
datasets: [
|
||||||
|
{
|
||||||
|
type: 'metrics',
|
||||||
|
name: 'endpoint.metadata',
|
||||||
|
title: 'Endpoint Metadata',
|
||||||
|
release: 'experimental',
|
||||||
|
package: 'endpoint',
|
||||||
|
ingest_pipeline: 'default',
|
||||||
|
elasticsearch: {
|
||||||
|
'index_template.mappings': {
|
||||||
|
dynamic: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
path: 'metadata',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
type: 'metrics',
|
||||||
|
name: 'endpoint.metadata_current',
|
||||||
|
title: 'Endpoint Metadata Current',
|
||||||
|
release: 'experimental',
|
||||||
|
package: 'endpoint',
|
||||||
|
ingest_pipeline: 'default',
|
||||||
|
elasticsearch: {
|
||||||
|
'index_template.mappings': {
|
||||||
|
dynamic: false,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
path: 'metadata_current',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
} as unknown) as RegistryPackage,
|
||||||
|
[],
|
||||||
|
legacyScopedClusterClient.callAsCurrentUser,
|
||||||
|
savedObjectsClient
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
ignore: [404],
|
||||||
|
method: 'POST',
|
||||||
|
path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0/_stop',
|
||||||
|
query: 'force=true',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
[
|
||||||
|
'transport.request',
|
||||||
|
{
|
||||||
|
ignore: [404],
|
||||||
|
method: 'DELETE',
|
||||||
|
path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0',
|
||||||
|
query: 'force=true',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
expect(savedObjectsClient.update.mock.calls).toEqual([
|
||||||
|
[
|
||||||
|
'epm-packages',
|
||||||
|
'endpoint',
|
||||||
|
{
|
||||||
|
installed_es: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
});
|
|
@ -35,6 +35,7 @@ import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
|
||||||
import { deleteKibanaSavedObjectsAssets } from './remove';
|
import { deleteKibanaSavedObjectsAssets } from './remove';
|
||||||
import { PackageOutdatedError } from '../../../errors';
|
import { PackageOutdatedError } from '../../../errors';
|
||||||
import { getPackageSavedObjects } from './get';
|
import { getPackageSavedObjects } from './get';
|
||||||
|
import { installTransformForDataset } from '../elasticsearch/transform/install';
|
||||||
|
|
||||||
export async function installLatestPackage(options: {
|
export async function installLatestPackage(options: {
|
||||||
savedObjectsClient: SavedObjectsClientContract;
|
savedObjectsClient: SavedObjectsClientContract;
|
||||||
|
@ -191,6 +192,13 @@ export async function installPackage({
|
||||||
// update current backing indices of each data stream
|
// update current backing indices of each data stream
|
||||||
await updateCurrentWriteIndices(callCluster, installedTemplates);
|
await updateCurrentWriteIndices(callCluster, installedTemplates);
|
||||||
|
|
||||||
|
const installedTransforms = await installTransformForDataset(
|
||||||
|
registryPackageInfo,
|
||||||
|
paths,
|
||||||
|
callCluster,
|
||||||
|
savedObjectsClient
|
||||||
|
);
|
||||||
|
|
||||||
// if this is an update or retrying an update, delete the previous version's pipelines
|
// if this is an update or retrying an update, delete the previous version's pipelines
|
||||||
if (installType === 'update' || installType === 'reupdate') {
|
if (installType === 'update' || installType === 'reupdate') {
|
||||||
await deletePreviousPipelines(
|
await deletePreviousPipelines(
|
||||||
|
@ -216,13 +224,19 @@ export async function installPackage({
|
||||||
type: ElasticsearchAssetType.indexTemplate,
|
type: ElasticsearchAssetType.indexTemplate,
|
||||||
}));
|
}));
|
||||||
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
|
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
|
||||||
|
|
||||||
// update to newly installed version when all assets are successfully installed
|
// update to newly installed version when all assets are successfully installed
|
||||||
if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
|
if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
|
||||||
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
|
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
|
||||||
install_version: pkgVersion,
|
install_version: pkgVersion,
|
||||||
install_status: 'installed',
|
install_status: 'installed',
|
||||||
});
|
});
|
||||||
return [...installedKibanaAssetsRefs, ...installedPipelines, ...installedTemplateRefs];
|
return [
|
||||||
|
...installedKibanaAssetsRefs,
|
||||||
|
...installedPipelines,
|
||||||
|
...installedTemplateRefs,
|
||||||
|
...installedTransforms,
|
||||||
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
const updateVersion = async (
|
const updateVersion = async (
|
||||||
|
|
|
@ -6,12 +6,17 @@
|
||||||
|
|
||||||
import { SavedObjectsClientContract } from 'src/core/server';
|
import { SavedObjectsClientContract } from 'src/core/server';
|
||||||
import Boom from 'boom';
|
import Boom from 'boom';
|
||||||
import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../constants';
|
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
|
||||||
import { AssetReference, AssetType, ElasticsearchAssetType } from '../../../types';
|
import {
|
||||||
import { CallESAsCurrentUser } from '../../../types';
|
AssetReference,
|
||||||
|
AssetType,
|
||||||
|
CallESAsCurrentUser,
|
||||||
|
ElasticsearchAssetType,
|
||||||
|
} from '../../../types';
|
||||||
import { getInstallation, savedObjectTypes } from './index';
|
import { getInstallation, savedObjectTypes } from './index';
|
||||||
import { deletePipeline } from '../elasticsearch/ingest_pipeline/';
|
import { deletePipeline } from '../elasticsearch/ingest_pipeline/';
|
||||||
import { installIndexPatterns } from '../kibana/index_pattern/install';
|
import { installIndexPatterns } from '../kibana/index_pattern/install';
|
||||||
|
import { deleteTransforms } from '../elasticsearch/transform/remove';
|
||||||
import { packagePolicyService, appContextService } from '../..';
|
import { packagePolicyService, appContextService } from '../..';
|
||||||
import { splitPkgKey, deletePackageCache, getArchiveInfo } from '../registry';
|
import { splitPkgKey, deletePackageCache, getArchiveInfo } from '../registry';
|
||||||
|
|
||||||
|
@ -72,6 +77,8 @@ async function deleteAssets(
|
||||||
return deletePipeline(callCluster, id);
|
return deletePipeline(callCluster, id);
|
||||||
} else if (assetType === ElasticsearchAssetType.indexTemplate) {
|
} else if (assetType === ElasticsearchAssetType.indexTemplate) {
|
||||||
return deleteTemplate(callCluster, id);
|
return deleteTemplate(callCluster, id);
|
||||||
|
} else if (assetType === ElasticsearchAssetType.transform) {
|
||||||
|
return deleteTransforms(callCluster, [id]);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
export const eventsIndexPattern = 'logs-endpoint.events.*';
|
export const eventsIndexPattern = 'logs-endpoint.events.*';
|
||||||
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
|
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
|
||||||
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
|
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
|
||||||
|
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current-*';
|
||||||
export const policyIndexPattern = 'metrics-endpoint.policy-*';
|
export const policyIndexPattern = 'metrics-endpoint.policy-*';
|
||||||
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
|
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
|
||||||
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';
|
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';
|
||||||
|
|
|
@ -445,6 +445,13 @@ export type HostInfo = Immutable<{
|
||||||
host_status: HostStatus;
|
host_status: HostStatus;
|
||||||
}>;
|
}>;
|
||||||
|
|
||||||
|
export type HostMetadataDetails = Immutable<{
|
||||||
|
agent: {
|
||||||
|
id: string;
|
||||||
|
};
|
||||||
|
HostDetails: HostMetadata;
|
||||||
|
}>;
|
||||||
|
|
||||||
export type HostMetadata = Immutable<{
|
export type HostMetadata = Immutable<{
|
||||||
'@timestamp': number;
|
'@timestamp': number;
|
||||||
event: {
|
event: {
|
||||||
|
|
|
@ -9,11 +9,12 @@ import { SearchResponse } from 'elasticsearch';
|
||||||
import { schema } from '@kbn/config-schema';
|
import { schema } from '@kbn/config-schema';
|
||||||
import Boom from 'boom';
|
import Boom from 'boom';
|
||||||
|
|
||||||
import { metadataIndexPattern } from '../../../../common/endpoint/constants';
|
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
|
||||||
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
|
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
|
||||||
import {
|
import {
|
||||||
HostInfo,
|
HostInfo,
|
||||||
HostMetadata,
|
HostMetadata,
|
||||||
|
HostMetadataDetails,
|
||||||
HostResultList,
|
HostResultList,
|
||||||
HostStatus,
|
HostStatus,
|
||||||
} from '../../../../common/endpoint/types';
|
} from '../../../../common/endpoint/types';
|
||||||
|
@ -23,10 +24,6 @@ import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/m
|
||||||
import { findAllUnenrolledAgentIds } from './support/unenroll';
|
import { findAllUnenrolledAgentIds } from './support/unenroll';
|
||||||
import { findAgentIDsByStatus } from './support/agent_status';
|
import { findAgentIDsByStatus } from './support/agent_status';
|
||||||
|
|
||||||
interface HitSource {
|
|
||||||
_source: HostMetadata;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface MetadataRequestContext {
|
interface MetadataRequestContext {
|
||||||
agentService: AgentService;
|
agentService: AgentService;
|
||||||
logger: Logger;
|
logger: Logger;
|
||||||
|
@ -127,7 +124,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
|
||||||
const queryParams = await kibanaRequestToMetadataListESQuery(
|
const queryParams = await kibanaRequestToMetadataListESQuery(
|
||||||
req,
|
req,
|
||||||
endpointAppContext,
|
endpointAppContext,
|
||||||
metadataIndexPattern,
|
metadataCurrentIndexPattern,
|
||||||
{
|
{
|
||||||
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
|
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
|
||||||
statusAgentIDs: statusIDs,
|
statusAgentIDs: statusIDs,
|
||||||
|
@ -137,7 +134,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
|
||||||
const response = (await context.core.elasticsearch.legacy.client.callAsCurrentUser(
|
const response = (await context.core.elasticsearch.legacy.client.callAsCurrentUser(
|
||||||
'search',
|
'search',
|
||||||
queryParams
|
queryParams
|
||||||
)) as SearchResponse<HostMetadata>;
|
)) as SearchResponse<HostMetadataDetails>;
|
||||||
|
|
||||||
return res.ok({
|
return res.ok({
|
||||||
body: await mapToHostResultList(queryParams, response, metadataRequestContext),
|
body: await mapToHostResultList(queryParams, response, metadataRequestContext),
|
||||||
|
@ -193,17 +190,17 @@ export async function getHostData(
|
||||||
metadataRequestContext: MetadataRequestContext,
|
metadataRequestContext: MetadataRequestContext,
|
||||||
id: string
|
id: string
|
||||||
): Promise<HostInfo | undefined> {
|
): Promise<HostInfo | undefined> {
|
||||||
const query = getESQueryHostMetadataByID(id, metadataIndexPattern);
|
const query = getESQueryHostMetadataByID(id, metadataCurrentIndexPattern);
|
||||||
const response = (await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser(
|
const response = (await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser(
|
||||||
'search',
|
'search',
|
||||||
query
|
query
|
||||||
)) as SearchResponse<HostMetadata>;
|
)) as SearchResponse<HostMetadataDetails>;
|
||||||
|
|
||||||
if (response.hits.hits.length === 0) {
|
if (response.hits.hits.length === 0) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
const hostMetadata: HostMetadata = response.hits.hits[0]._source;
|
const hostMetadata: HostMetadata = response.hits.hits[0]._source.HostDetails;
|
||||||
const agent = await findAgent(metadataRequestContext, hostMetadata);
|
const agent = await findAgent(metadataRequestContext, hostMetadata);
|
||||||
|
|
||||||
if (agent && !agent.active) {
|
if (agent && !agent.active) {
|
||||||
|
@ -241,19 +238,19 @@ async function findAgent(
|
||||||
async function mapToHostResultList(
|
async function mapToHostResultList(
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
queryParams: Record<string, any>,
|
queryParams: Record<string, any>,
|
||||||
searchResponse: SearchResponse<HostMetadata>,
|
searchResponse: SearchResponse<HostMetadataDetails>,
|
||||||
metadataRequestContext: MetadataRequestContext
|
metadataRequestContext: MetadataRequestContext
|
||||||
): Promise<HostResultList> {
|
): Promise<HostResultList> {
|
||||||
const totalNumberOfHosts = searchResponse?.aggregations?.total?.value || 0;
|
const totalNumberOfHosts =
|
||||||
|
((searchResponse.hits?.total as unknown) as { value: number; relation: string }).value || 0;
|
||||||
if (searchResponse.hits.hits.length > 0) {
|
if (searchResponse.hits.hits.length > 0) {
|
||||||
return {
|
return {
|
||||||
request_page_size: queryParams.size,
|
request_page_size: queryParams.size,
|
||||||
request_page_index: queryParams.from,
|
request_page_index: queryParams.from,
|
||||||
hosts: await Promise.all(
|
hosts: await Promise.all(
|
||||||
searchResponse.hits.hits
|
searchResponse.hits.hits.map(async (entry) =>
|
||||||
.map((response) => response.inner_hits.most_recent.hits.hits)
|
enrichHostMetadata(entry._source.HostDetails, metadataRequestContext)
|
||||||
.flatMap((data) => data as HitSource)
|
)
|
||||||
.map(async (entry) => enrichHostMetadata(entry._source, metadataRequestContext))
|
|
||||||
),
|
),
|
||||||
total: totalNumberOfHosts,
|
total: totalNumberOfHosts,
|
||||||
};
|
};
|
||||||
|
|
|
@ -23,6 +23,7 @@ import {
|
||||||
import {
|
import {
|
||||||
HostInfo,
|
HostInfo,
|
||||||
HostMetadata,
|
HostMetadata,
|
||||||
|
HostMetadataDetails,
|
||||||
HostResultList,
|
HostResultList,
|
||||||
HostStatus,
|
HostStatus,
|
||||||
} from '../../../../common/endpoint/types';
|
} from '../../../../common/endpoint/types';
|
||||||
|
@ -141,7 +142,7 @@ describe('test endpoint route', () => {
|
||||||
bool: {
|
bool: {
|
||||||
must_not: {
|
must_not: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': [
|
'HostDetails.elastic.agent.id': [
|
||||||
'00000000-0000-0000-0000-000000000000',
|
'00000000-0000-0000-0000-000000000000',
|
||||||
'11111111-1111-1111-1111-111111111111',
|
'11111111-1111-1111-1111-111111111111',
|
||||||
],
|
],
|
||||||
|
@ -197,7 +198,7 @@ describe('test endpoint route', () => {
|
||||||
bool: {
|
bool: {
|
||||||
must_not: {
|
must_not: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': [
|
'HostDetails.elastic.agent.id': [
|
||||||
'00000000-0000-0000-0000-000000000000',
|
'00000000-0000-0000-0000-000000000000',
|
||||||
'11111111-1111-1111-1111-111111111111',
|
'11111111-1111-1111-1111-111111111111',
|
||||||
],
|
],
|
||||||
|
@ -442,7 +443,7 @@ describe('Filters Schema Test', () => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadata> {
|
function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadataDetails> {
|
||||||
return ({
|
return ({
|
||||||
took: 15,
|
took: 15,
|
||||||
timed_out: false,
|
timed_out: false,
|
||||||
|
@ -454,7 +455,7 @@ function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostM
|
||||||
},
|
},
|
||||||
hits: {
|
hits: {
|
||||||
total: {
|
total: {
|
||||||
value: 5,
|
value: 1,
|
||||||
relation: 'eq',
|
relation: 'eq',
|
||||||
},
|
},
|
||||||
max_score: null,
|
max_score: null,
|
||||||
|
@ -464,36 +465,18 @@ function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostM
|
||||||
_index: 'metrics-endpoint.metadata-default',
|
_index: 'metrics-endpoint.metadata-default',
|
||||||
_id: '8FhM0HEBYyRTvb6lOQnw',
|
_id: '8FhM0HEBYyRTvb6lOQnw',
|
||||||
_score: null,
|
_score: null,
|
||||||
_source: hostMetadata,
|
_source: {
|
||||||
sort: [1588337587997],
|
agent: {
|
||||||
inner_hits: {
|
id: '1e3472bb-5c20-4946-b469-b5af1a809e4f',
|
||||||
most_recent: {
|
},
|
||||||
hits: {
|
HostDetails: {
|
||||||
total: {
|
...hostMetadata,
|
||||||
value: 2,
|
|
||||||
relation: 'eq',
|
|
||||||
},
|
|
||||||
max_score: null,
|
|
||||||
hits: [
|
|
||||||
{
|
|
||||||
_index: 'metrics-endpoint.metadata-default',
|
|
||||||
_id: 'W6Vo1G8BYQH1gtPUgYkC',
|
|
||||||
_score: null,
|
|
||||||
_source: hostMetadata,
|
|
||||||
sort: [1579816615336],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
sort: [1588337587997],
|
||||||
},
|
},
|
||||||
]
|
]
|
||||||
: [],
|
: [],
|
||||||
},
|
},
|
||||||
aggregations: {
|
} as unknown) as SearchResponse<HostMetadataDetails>;
|
||||||
total: {
|
|
||||||
value: 1,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
} as unknown) as SearchResponse<HostMetadata>;
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,7 @@ import { httpServerMock, loggingSystemMock } from '../../../../../../../src/core
|
||||||
import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders';
|
import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders';
|
||||||
import { EndpointAppContextService } from '../../endpoint_app_context_services';
|
import { EndpointAppContextService } from '../../endpoint_app_context_services';
|
||||||
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
|
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
|
||||||
import { metadataIndexPattern } from '../../../../common/endpoint/constants';
|
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
|
||||||
|
|
||||||
describe('query builder', () => {
|
describe('query builder', () => {
|
||||||
describe('MetadataListESQuery', () => {
|
describe('MetadataListESQuery', () => {
|
||||||
|
@ -22,31 +22,16 @@ describe('query builder', () => {
|
||||||
service: new EndpointAppContextService(),
|
service: new EndpointAppContextService(),
|
||||||
config: () => Promise.resolve(createMockConfig()),
|
config: () => Promise.resolve(createMockConfig()),
|
||||||
},
|
},
|
||||||
metadataIndexPattern
|
metadataCurrentIndexPattern
|
||||||
);
|
);
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
body: {
|
body: {
|
||||||
query: {
|
query: {
|
||||||
match_all: {},
|
match_all: {},
|
||||||
},
|
},
|
||||||
collapse: {
|
|
||||||
field: 'host.id',
|
|
||||||
inner_hits: {
|
|
||||||
name: 'most_recent',
|
|
||||||
size: 1,
|
|
||||||
sort: [{ 'event.created': 'desc' }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
total: {
|
|
||||||
cardinality: {
|
|
||||||
field: 'host.id',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -54,7 +39,7 @@ describe('query builder', () => {
|
||||||
},
|
},
|
||||||
from: 0,
|
from: 0,
|
||||||
size: 10,
|
size: 10,
|
||||||
index: metadataIndexPattern,
|
index: metadataCurrentIndexPattern,
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
} as Record<string, any>);
|
} as Record<string, any>);
|
||||||
});
|
});
|
||||||
|
@ -74,7 +59,7 @@ describe('query builder', () => {
|
||||||
service: new EndpointAppContextService(),
|
service: new EndpointAppContextService(),
|
||||||
config: () => Promise.resolve(createMockConfig()),
|
config: () => Promise.resolve(createMockConfig()),
|
||||||
},
|
},
|
||||||
metadataIndexPattern,
|
metadataCurrentIndexPattern,
|
||||||
{
|
{
|
||||||
unenrolledAgentIds: [unenrolledElasticAgentId],
|
unenrolledAgentIds: [unenrolledElasticAgentId],
|
||||||
}
|
}
|
||||||
|
@ -86,29 +71,14 @@ describe('query builder', () => {
|
||||||
bool: {
|
bool: {
|
||||||
must_not: {
|
must_not: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': [unenrolledElasticAgentId],
|
'HostDetails.elastic.agent.id': [unenrolledElasticAgentId],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
collapse: {
|
|
||||||
field: 'host.id',
|
|
||||||
inner_hits: {
|
|
||||||
name: 'most_recent',
|
|
||||||
size: 1,
|
|
||||||
sort: [{ 'event.created': 'desc' }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
total: {
|
|
||||||
cardinality: {
|
|
||||||
field: 'host.id',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -116,7 +86,7 @@ describe('query builder', () => {
|
||||||
},
|
},
|
||||||
from: 0,
|
from: 0,
|
||||||
size: 10,
|
size: 10,
|
||||||
index: metadataIndexPattern,
|
index: metadataCurrentIndexPattern,
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
} as Record<string, any>);
|
} as Record<string, any>);
|
||||||
}
|
}
|
||||||
|
@ -127,7 +97,7 @@ describe('query builder', () => {
|
||||||
it('test default query params for all endpoints metadata when body filter is provided', async () => {
|
it('test default query params for all endpoints metadata when body filter is provided', async () => {
|
||||||
const mockRequest = httpServerMock.createKibanaRequest({
|
const mockRequest = httpServerMock.createKibanaRequest({
|
||||||
body: {
|
body: {
|
||||||
filters: { kql: 'not host.ip:10.140.73.246' },
|
filters: { kql: 'not HostDetails.host.ip:10.140.73.246' },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const query = await kibanaRequestToMetadataListESQuery(
|
const query = await kibanaRequestToMetadataListESQuery(
|
||||||
|
@ -137,7 +107,7 @@ describe('query builder', () => {
|
||||||
service: new EndpointAppContextService(),
|
service: new EndpointAppContextService(),
|
||||||
config: () => Promise.resolve(createMockConfig()),
|
config: () => Promise.resolve(createMockConfig()),
|
||||||
},
|
},
|
||||||
metadataIndexPattern
|
metadataCurrentIndexPattern
|
||||||
);
|
);
|
||||||
|
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
|
@ -152,7 +122,7 @@ describe('query builder', () => {
|
||||||
should: [
|
should: [
|
||||||
{
|
{
|
||||||
match: {
|
match: {
|
||||||
'host.ip': '10.140.73.246',
|
'HostDetails.host.ip': '10.140.73.246',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
@ -164,24 +134,9 @@ describe('query builder', () => {
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
collapse: {
|
|
||||||
field: 'host.id',
|
|
||||||
inner_hits: {
|
|
||||||
name: 'most_recent',
|
|
||||||
size: 1,
|
|
||||||
sort: [{ 'event.created': 'desc' }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
total: {
|
|
||||||
cardinality: {
|
|
||||||
field: 'host.id',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -189,7 +144,7 @@ describe('query builder', () => {
|
||||||
},
|
},
|
||||||
from: 0,
|
from: 0,
|
||||||
size: 10,
|
size: 10,
|
||||||
index: metadataIndexPattern,
|
index: metadataCurrentIndexPattern,
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
} as Record<string, any>);
|
} as Record<string, any>);
|
||||||
});
|
});
|
||||||
|
@ -201,7 +156,7 @@ describe('query builder', () => {
|
||||||
const unenrolledElasticAgentId = '1fdca33f-799f-49f4-939c-ea4383c77672';
|
const unenrolledElasticAgentId = '1fdca33f-799f-49f4-939c-ea4383c77672';
|
||||||
const mockRequest = httpServerMock.createKibanaRequest({
|
const mockRequest = httpServerMock.createKibanaRequest({
|
||||||
body: {
|
body: {
|
||||||
filters: { kql: 'not host.ip:10.140.73.246' },
|
filters: { kql: 'not HostDetails.host.ip:10.140.73.246' },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
const query = await kibanaRequestToMetadataListESQuery(
|
const query = await kibanaRequestToMetadataListESQuery(
|
||||||
|
@ -211,7 +166,7 @@ describe('query builder', () => {
|
||||||
service: new EndpointAppContextService(),
|
service: new EndpointAppContextService(),
|
||||||
config: () => Promise.resolve(createMockConfig()),
|
config: () => Promise.resolve(createMockConfig()),
|
||||||
},
|
},
|
||||||
metadataIndexPattern,
|
metadataCurrentIndexPattern,
|
||||||
{
|
{
|
||||||
unenrolledAgentIds: [unenrolledElasticAgentId],
|
unenrolledAgentIds: [unenrolledElasticAgentId],
|
||||||
}
|
}
|
||||||
|
@ -226,7 +181,7 @@ describe('query builder', () => {
|
||||||
bool: {
|
bool: {
|
||||||
must_not: {
|
must_not: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': [unenrolledElasticAgentId],
|
'HostDetails.elastic.agent.id': [unenrolledElasticAgentId],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -238,7 +193,7 @@ describe('query builder', () => {
|
||||||
should: [
|
should: [
|
||||||
{
|
{
|
||||||
match: {
|
match: {
|
||||||
'host.ip': '10.140.73.246',
|
'HostDetails.host.ip': '10.140.73.246',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
|
@ -250,24 +205,9 @@ describe('query builder', () => {
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
collapse: {
|
|
||||||
field: 'host.id',
|
|
||||||
inner_hits: {
|
|
||||||
name: 'most_recent',
|
|
||||||
size: 1,
|
|
||||||
sort: [{ 'event.created': 'desc' }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
total: {
|
|
||||||
cardinality: {
|
|
||||||
field: 'host.id',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -275,7 +215,7 @@ describe('query builder', () => {
|
||||||
},
|
},
|
||||||
from: 0,
|
from: 0,
|
||||||
size: 10,
|
size: 10,
|
||||||
index: metadataIndexPattern,
|
index: metadataCurrentIndexPattern,
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
} as Record<string, any>);
|
} as Record<string, any>);
|
||||||
}
|
}
|
||||||
|
@ -285,15 +225,15 @@ describe('query builder', () => {
|
||||||
describe('MetadataGetQuery', () => {
|
describe('MetadataGetQuery', () => {
|
||||||
it('searches for the correct ID', () => {
|
it('searches for the correct ID', () => {
|
||||||
const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';
|
const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';
|
||||||
const query = getESQueryHostMetadataByID(mockID, metadataIndexPattern);
|
const query = getESQueryHostMetadataByID(mockID, metadataCurrentIndexPattern);
|
||||||
|
|
||||||
expect(query).toEqual({
|
expect(query).toEqual({
|
||||||
body: {
|
body: {
|
||||||
query: { match: { 'host.id': mockID } },
|
query: { match: { 'HostDetails.host.id': mockID } },
|
||||||
sort: [{ 'event.created': { order: 'desc' } }],
|
sort: [{ 'HostDetails.event.created': { order: 'desc' } }],
|
||||||
size: 1,
|
size: 1,
|
||||||
},
|
},
|
||||||
index: metadataIndexPattern,
|
index: metadataCurrentIndexPattern,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
|
@ -28,24 +28,9 @@ export async function kibanaRequestToMetadataListESQuery(
|
||||||
queryBuilderOptions?.unenrolledAgentIds!,
|
queryBuilderOptions?.unenrolledAgentIds!,
|
||||||
queryBuilderOptions?.statusAgentIDs!
|
queryBuilderOptions?.statusAgentIDs!
|
||||||
),
|
),
|
||||||
collapse: {
|
|
||||||
field: 'host.id',
|
|
||||||
inner_hits: {
|
|
||||||
name: 'most_recent',
|
|
||||||
size: 1,
|
|
||||||
sort: [{ 'event.created': 'desc' }],
|
|
||||||
},
|
|
||||||
},
|
|
||||||
aggs: {
|
|
||||||
total: {
|
|
||||||
cardinality: {
|
|
||||||
field: 'host.id',
|
|
||||||
},
|
|
||||||
},
|
|
||||||
},
|
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
@ -90,7 +75,7 @@ function buildQueryBody(
|
||||||
? {
|
? {
|
||||||
must_not: {
|
must_not: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': unerolledAgentIds,
|
'HostDetails.elastic.agent.id': unerolledAgentIds,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -99,7 +84,7 @@ function buildQueryBody(
|
||||||
? {
|
? {
|
||||||
must: {
|
must: {
|
||||||
terms: {
|
terms: {
|
||||||
'elastic.agent.id': statusAgentIDs,
|
'HostDetails.elastic.agent.id': statusAgentIDs,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -137,12 +122,12 @@ export function getESQueryHostMetadataByID(hostID: string, index: string) {
|
||||||
body: {
|
body: {
|
||||||
query: {
|
query: {
|
||||||
match: {
|
match: {
|
||||||
'host.id': hostID,
|
'HostDetails.host.id': hostID,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
sort: [
|
sort: [
|
||||||
{
|
{
|
||||||
'event.created': {
|
'HostDetails.event.created': {
|
||||||
order: 'desc',
|
order: 'desc',
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|
|
@ -84,6 +84,13 @@ export default function (providerContext: FtrProviderContext) {
|
||||||
});
|
});
|
||||||
expect(resSettings.statusCode).equal(200);
|
expect(resSettings.statusCode).equal(200);
|
||||||
});
|
});
|
||||||
|
it('should have installed the transform components', async function () {
|
||||||
|
const res = await es.transport.request({
|
||||||
|
method: 'GET',
|
||||||
|
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
|
||||||
|
});
|
||||||
|
expect(res.statusCode).equal(200);
|
||||||
|
});
|
||||||
it('should have installed the kibana assets', async function () {
|
it('should have installed the kibana assets', async function () {
|
||||||
const resIndexPatternLogs = await kibanaServer.savedObjects.get({
|
const resIndexPatternLogs = await kibanaServer.savedObjects.get({
|
||||||
type: 'index-pattern',
|
type: 'index-pattern',
|
||||||
|
@ -161,6 +168,10 @@ export default function (providerContext: FtrProviderContext) {
|
||||||
id: 'metrics-all_assets.test_metrics',
|
id: 'metrics-all_assets.test_metrics',
|
||||||
type: 'index_template',
|
type: 'index_template',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: 'logs-all_assets.test_logs-default-0.1.0',
|
||||||
|
type: 'transform',
|
||||||
|
},
|
||||||
],
|
],
|
||||||
es_index_patterns: {
|
es_index_patterns: {
|
||||||
test_logs: 'logs-all_assets.test_logs-*',
|
test_logs: 'logs-all_assets.test_logs-*',
|
||||||
|
@ -237,6 +248,18 @@ export default function (providerContext: FtrProviderContext) {
|
||||||
);
|
);
|
||||||
expect(resPipeline2.statusCode).equal(404);
|
expect(resPipeline2.statusCode).equal(404);
|
||||||
});
|
});
|
||||||
|
it('should have uninstalled the transforms', async function () {
|
||||||
|
const res = await es.transport.request(
|
||||||
|
{
|
||||||
|
method: 'GET',
|
||||||
|
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ignore: [404],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
expect(res.statusCode).equal(404);
|
||||||
|
});
|
||||||
it('should have uninstalled the kibana assets', async function () {
|
it('should have uninstalled the kibana assets', async function () {
|
||||||
let resDashboard;
|
let resDashboard;
|
||||||
try {
|
try {
|
||||||
|
|
|
@ -0,0 +1,35 @@
|
||||||
|
{
|
||||||
|
"source": {
|
||||||
|
"index": "logs-all_assets.test_log-default*"
|
||||||
|
},
|
||||||
|
"dest": {
|
||||||
|
"index": "logs-all_assets.test_log_current-default"
|
||||||
|
},
|
||||||
|
"pivot": {
|
||||||
|
"group_by": {
|
||||||
|
"agent.id": {
|
||||||
|
"terms": {
|
||||||
|
"field": "agent.id"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"aggregations": {
|
||||||
|
"HostDetails": {
|
||||||
|
"scripted_metric": {
|
||||||
|
"init_script": "state.timestamp_latest = 0L; state.last_doc=''",
|
||||||
|
"map_script": "def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli(); if (current_date \u003e state.timestamp_latest) {state.timestamp_latest = current_date;state.last_doc = new HashMap(params['_source']);}",
|
||||||
|
"combine_script": "return state",
|
||||||
|
"reduce_script": "def last_doc = '';def timestamp_latest = 0L; for (s in states) {if (s.timestamp_latest \u003e (timestamp_latest)) {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}} return last_doc"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "collapse and update the latest document for each host",
|
||||||
|
"frequency": "1m",
|
||||||
|
"sync": {
|
||||||
|
"time": {
|
||||||
|
"field": "event.ingested",
|
||||||
|
"delay": "60s"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -12,7 +12,7 @@ import { defineDockerServersConfig } from '@kbn/test';
|
||||||
// Docker image to use for Ingest Manager API integration tests.
|
// Docker image to use for Ingest Manager API integration tests.
|
||||||
// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit
|
// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit
|
||||||
export const dockerImage =
|
export const dockerImage =
|
||||||
'docker.elastic.co/package-registry/distribution:f6b01daec8cfe355101e366de9941d35a4c3763e';
|
'docker.elastic.co/package-registry/distribution:5e0e12ce1bc2cb0c2f67f2e07d11b9a6043bcf25';
|
||||||
|
|
||||||
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
|
||||||
const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
|
const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));
|
||||||
|
|
|
@ -6,7 +6,12 @@
|
||||||
|
|
||||||
import expect from '@kbn/expect';
|
import expect from '@kbn/expect';
|
||||||
import { FtrProviderContext } from '../../ftr_provider_context';
|
import { FtrProviderContext } from '../../ftr_provider_context';
|
||||||
import { deleteMetadataStream } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
|
|
||||||
|
import {
|
||||||
|
deleteMetadataCurrentStream,
|
||||||
|
deleteMetadataStream,
|
||||||
|
} from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
|
||||||
|
|
||||||
export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);
|
const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);
|
||||||
const esArchiver = getService('esArchiver');
|
const esArchiver = getService('esArchiver');
|
||||||
|
@ -23,6 +28,16 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
'Version',
|
'Version',
|
||||||
'Last Active',
|
'Last Active',
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
'rezzani-7.example.com',
|
||||||
|
'Error',
|
||||||
|
'Default',
|
||||||
|
'Failure',
|
||||||
|
'windows 10.0',
|
||||||
|
'10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c',
|
||||||
|
'6.8.0',
|
||||||
|
'Jan 24, 2020 @ 16:06:09.541',
|
||||||
|
],
|
||||||
[
|
[
|
||||||
'cadmann-4.example.com',
|
'cadmann-4.example.com',
|
||||||
'Error',
|
'Error',
|
||||||
|
@ -43,16 +58,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
'6.0.0',
|
'6.0.0',
|
||||||
'Jan 24, 2020 @ 16:06:09.541',
|
'Jan 24, 2020 @ 16:06:09.541',
|
||||||
],
|
],
|
||||||
[
|
|
||||||
'rezzani-7.example.com',
|
|
||||||
'Error',
|
|
||||||
'Default',
|
|
||||||
'Failure',
|
|
||||||
'windows 10.0',
|
|
||||||
'10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c',
|
|
||||||
'6.8.0',
|
|
||||||
'Jan 24, 2020 @ 16:06:09.541',
|
|
||||||
],
|
|
||||||
];
|
];
|
||||||
|
|
||||||
describe('endpoint list', function () {
|
describe('endpoint list', function () {
|
||||||
|
@ -61,10 +66,13 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
|
|
||||||
describe('when initially navigating to page', () => {
|
describe('when initially navigating to page', () => {
|
||||||
before(async () => {
|
before(async () => {
|
||||||
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
await pageObjects.endpoint.navigateToEndpointList();
|
await pageObjects.endpoint.navigateToEndpointList();
|
||||||
});
|
});
|
||||||
after(async () => {
|
after(async () => {
|
||||||
await deleteMetadataStream(getService);
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('finds no data in list and prompts onboarding to add policy', async () => {
|
it('finds no data in list and prompts onboarding to add policy', async () => {
|
||||||
|
@ -73,7 +81,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
|
|
||||||
it('finds data after load and polling', async () => {
|
it('finds data after load and polling', async () => {
|
||||||
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
|
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
|
||||||
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 10000);
|
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 120000);
|
||||||
const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable');
|
const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable');
|
||||||
expect(tableData).to.eql(expectedData);
|
expect(tableData).to.eql(expectedData);
|
||||||
});
|
});
|
||||||
|
@ -82,10 +90,12 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
describe('when there is data,', () => {
|
describe('when there is data,', () => {
|
||||||
before(async () => {
|
before(async () => {
|
||||||
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
|
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
|
||||||
|
await sleep(120000);
|
||||||
await pageObjects.endpoint.navigateToEndpointList();
|
await pageObjects.endpoint.navigateToEndpointList();
|
||||||
});
|
});
|
||||||
after(async () => {
|
after(async () => {
|
||||||
await deleteMetadataStream(getService);
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('finds page title', async () => {
|
it('finds page title', async () => {
|
||||||
|
@ -202,10 +212,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('when there is no data,', () => {
|
describe.skip('when there is no data,', () => {
|
||||||
before(async () => {
|
before(async () => {
|
||||||
// clear out the data and reload the page
|
// clear out the data and reload the page
|
||||||
await deleteMetadataStream(getService);
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
await pageObjects.endpoint.navigateToEndpointList();
|
await pageObjects.endpoint.navigateToEndpointList();
|
||||||
});
|
});
|
||||||
it('displays empty Policy Table page.', async () => {
|
it('displays empty Policy Table page.', async () => {
|
||||||
|
|
|
@ -10,6 +10,7 @@ import {
|
||||||
eventsIndexPattern,
|
eventsIndexPattern,
|
||||||
alertsIndexPattern,
|
alertsIndexPattern,
|
||||||
policyIndexPattern,
|
policyIndexPattern,
|
||||||
|
metadataCurrentIndexPattern,
|
||||||
} from '../../../plugins/security_solution/common/endpoint/constants';
|
} from '../../../plugins/security_solution/common/endpoint/constants';
|
||||||
|
|
||||||
export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
|
export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
|
||||||
|
@ -25,10 +26,44 @@ export async function deleteDataStream(getService: (serviceName: 'es') => Client
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function deleteAllDocsFromIndex(
|
||||||
|
getService: (serviceName: 'es') => Client,
|
||||||
|
index: string
|
||||||
|
) {
|
||||||
|
const client = getService('es');
|
||||||
|
await client.deleteByQuery(
|
||||||
|
{
|
||||||
|
body: {
|
||||||
|
query: {
|
||||||
|
match_all: {},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
index: `${index}`,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ignore: [404],
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
|
export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
|
||||||
await deleteDataStream(getService, metadataIndexPattern);
|
await deleteDataStream(getService, metadataIndexPattern);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function deleteMetadataCurrentStream(getService: (serviceName: 'es') => Client) {
|
||||||
|
await deleteDataStream(getService, metadataCurrentIndexPattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteAllDocsFromMetadataIndex(getService: (serviceName: 'es') => Client) {
|
||||||
|
await deleteAllDocsFromIndex(getService, metadataIndexPattern);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteAllDocsFromMetadataCurrentIndex(
|
||||||
|
getService: (serviceName: 'es') => Client
|
||||||
|
) {
|
||||||
|
await deleteAllDocsFromIndex(getService, metadataCurrentIndexPattern);
|
||||||
|
}
|
||||||
|
|
||||||
export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
|
export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
|
||||||
await deleteDataStream(getService, eventsIndexPattern);
|
await deleteDataStream(getService, eventsIndexPattern);
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,12 @@
|
||||||
*/
|
*/
|
||||||
import expect from '@kbn/expect/expect.js';
|
import expect from '@kbn/expect/expect.js';
|
||||||
import { FtrProviderContext } from '../ftr_provider_context';
|
import { FtrProviderContext } from '../ftr_provider_context';
|
||||||
import { deleteMetadataStream } from './data_stream_helper';
|
import {
|
||||||
|
deleteAllDocsFromMetadataCurrentIndex,
|
||||||
|
deleteMetadataCurrentStream,
|
||||||
|
deleteAllDocsFromMetadataIndex,
|
||||||
|
deleteMetadataStream,
|
||||||
|
} from './data_stream_helper';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The number of host documents in the es archive.
|
* The number of host documents in the es archive.
|
||||||
|
@ -15,12 +20,14 @@ const numberOfHostsInFixture = 3;
|
||||||
export default function ({ getService }: FtrProviderContext) {
|
export default function ({ getService }: FtrProviderContext) {
|
||||||
const esArchiver = getService('esArchiver');
|
const esArchiver = getService('esArchiver');
|
||||||
const supertest = getService('supertest');
|
const supertest = getService('supertest');
|
||||||
|
|
||||||
describe('test metadata api', () => {
|
describe('test metadata api', () => {
|
||||||
describe('POST /api/endpoint/metadata when index is empty', () => {
|
describe('POST /api/endpoint/metadata when index is empty', () => {
|
||||||
it('metadata api should return empty result when index is empty', async () => {
|
it('metadata api should return empty result when index is empty', async () => {
|
||||||
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
|
|
||||||
// to do it manually
|
|
||||||
await deleteMetadataStream(getService);
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteAllDocsFromMetadataIndex(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
|
await deleteAllDocsFromMetadataCurrentIndex(getService);
|
||||||
const { body } = await supertest
|
const { body } = await supertest
|
||||||
.post('/api/endpoint/metadata')
|
.post('/api/endpoint/metadata')
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
|
@ -34,12 +41,19 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('POST /api/endpoint/metadata when index is not empty', () => {
|
describe('POST /api/endpoint/metadata when index is not empty', () => {
|
||||||
before(
|
before(async () => {
|
||||||
async () => await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true })
|
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
|
||||||
);
|
// wait for transform
|
||||||
|
await new Promise((r) => setTimeout(r, 120000));
|
||||||
|
});
|
||||||
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
|
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
|
||||||
// to do it manually
|
// to do it manually
|
||||||
after(async () => await deleteMetadataStream(getService));
|
after(async () => {
|
||||||
|
await deleteMetadataStream(getService);
|
||||||
|
await deleteAllDocsFromMetadataIndex(getService);
|
||||||
|
await deleteMetadataCurrentStream(getService);
|
||||||
|
await deleteAllDocsFromMetadataCurrentIndex(getService);
|
||||||
|
});
|
||||||
it('metadata api should return one entry for each host with default paging', async () => {
|
it('metadata api should return one entry for each host with default paging', async () => {
|
||||||
const { body } = await supertest
|
const { body } = await supertest
|
||||||
.post('/api/endpoint/metadata')
|
.post('/api/endpoint/metadata')
|
||||||
|
@ -121,7 +135,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
.send({
|
.send({
|
||||||
filters: {
|
filters: {
|
||||||
kql: 'not host.ip:10.46.229.234',
|
kql: 'not HostDetails.host.ip:10.46.229.234',
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
@ -146,7 +160,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
filters: {
|
filters: {
|
||||||
kql: `not host.ip:${notIncludedIp}`,
|
kql: `not HostDetails.host.ip:${notIncludedIp}`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
@ -154,12 +168,14 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
const resultIps: string[] = [].concat(
|
const resultIps: string[] = [].concat(
|
||||||
...body.hosts.map((hostInfo: Record<string, any>) => hostInfo.metadata.host.ip)
|
...body.hosts.map((hostInfo: Record<string, any>) => hostInfo.metadata.host.ip)
|
||||||
);
|
);
|
||||||
expect(resultIps).to.eql([
|
expect(resultIps.sort()).to.eql(
|
||||||
'10.192.213.130',
|
[
|
||||||
'10.70.28.129',
|
'10.192.213.130',
|
||||||
'10.101.149.26',
|
'10.70.28.129',
|
||||||
'2606:a000:ffc0:39:11ef:37b9:3371:578c',
|
'10.101.149.26',
|
||||||
]);
|
'2606:a000:ffc0:39:11ef:37b9:3371:578c',
|
||||||
|
].sort()
|
||||||
|
);
|
||||||
expect(resultIps).not.include.eql(notIncludedIp);
|
expect(resultIps).not.include.eql(notIncludedIp);
|
||||||
expect(body.hosts.length).to.eql(2);
|
expect(body.hosts.length).to.eql(2);
|
||||||
expect(body.request_page_size).to.eql(10);
|
expect(body.request_page_size).to.eql(10);
|
||||||
|
@ -173,7 +189,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
.send({
|
.send({
|
||||||
filters: {
|
filters: {
|
||||||
kql: `host.os.Ext.variant:${variantValue}`,
|
kql: `HostDetails.host.os.Ext.variant:${variantValue}`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
@ -194,7 +210,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
.send({
|
.send({
|
||||||
filters: {
|
filters: {
|
||||||
kql: `host.ip:${targetEndpointIp}`,
|
kql: `HostDetails.host.ip:${targetEndpointIp}`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
@ -215,7 +231,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
.send({
|
.send({
|
||||||
filters: {
|
filters: {
|
||||||
kql: `not Endpoint.policy.applied.status:success`,
|
kql: `not HostDetails.Endpoint.policy.applied.status:success`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
@ -236,7 +252,7 @@ export default function ({ getService }: FtrProviderContext) {
|
||||||
.set('kbn-xsrf', 'xxx')
|
.set('kbn-xsrf', 'xxx')
|
||||||
.send({
|
.send({
|
||||||
filters: {
|
filters: {
|
||||||
kql: `elastic.agent.id:${targetElasticAgentId}`,
|
kql: `HostDetails.elastic.agent.id:${targetElasticAgentId}`,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
.expect(200);
|
.expect(200);
|
||||||
|
|
Loading…
Reference in a new issue