maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SecuritySolution-Ingest]: use new metadata current and add transform installation (#74394)

Browse source 
[SecuritySolution-Ingest]: use new metadata current and add transform installation
This commit is contained in:
nnamdifrankie 2020-09-08 17:56:38 -04:00 committed by GitHub
parent 194d0b0a3f
commit 2ed4b57776
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 965 additions and 186 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/plugins/ingest_manager/common/types/models/epm.ts
View file

@ -40,6 +40,7 @@ export enum ElasticsearchAssetType {
ingestPipeline = 'ingest_pipeline',
indexTemplate = 'index_template',
ilmPolicy = 'ilm_policy',
transform = 'transform',
}
export enum AgentAssetType {

1
x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/epm/constants.tsx
View file

@ -19,6 +19,7 @@ export const AssetTitleMap: Record<AssetType, string> = {
dashboard: 'Dashboard',
ilm_policy: 'ILM Policy',
ingest_pipeline: 'Ingest Pipeline',
transform: 'Transform',
'index-pattern': 'Index Pattern',
index_template: 'Index Template',
component_template: 'Component Template',

11
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/common.ts Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import * as Registry from '../../registry';
export const getAsset = (path: string): Buffer => {
return Registry.getAsset(path);
};

165
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts Normal file
View file

@ -0,0 +1,165 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsClientContract } from 'kibana/server';
import { saveInstalledEsRefs } from '../../packages/install';
import * as Registry from '../../registry';
import {
Dataset,
ElasticsearchAssetType,
EsAssetReference,
RegistryPackage,
} from '../../../../../common/types/models';
import { CallESAsCurrentUser } from '../../../../types';
import { getInstallation } from '../../packages';
import { deleteTransforms, deleteTransformRefs } from './remove';
import { getAsset } from './common';
interface TransformInstallation {
installationName: string;
content: string;
}
interface TransformPathDataset {
path: string;
dataset: Dataset;
}
export const installTransformForDataset = async (
registryPackage: RegistryPackage,
paths: string[],
callCluster: CallESAsCurrentUser,
savedObjectsClient: SavedObjectsClientContract
) => {
const installation = await getInstallation({ savedObjectsClient, pkgName: registryPackage.name });
let previousInstalledTransformEsAssets: EsAssetReference[] = [];
if (installation) {
previousInstalledTransformEsAssets = installation.installed_es.filter(
({ type, id }) => type === ElasticsearchAssetType.transform
);
}
// delete all previous transform
await deleteTransforms(
callCluster,
previousInstalledTransformEsAssets.map((asset) => asset.id)
);
// install the latest dataset
const datasets = registryPackage.datasets;
if (!datasets?.length) return [];
const installNameSuffix = `${registryPackage.version}`;
const transformPaths = paths.filter((path) => isTransform(path));
let installedTransforms: EsAssetReference[] = [];
if (transformPaths.length > 0) {
const transformPathDatasets = datasets.reduce<TransformPathDataset[]>((acc, dataset) => {
transformPaths.forEach((path) => {
if (isDatasetTransform(path, dataset.path)) {
acc.push({ path, dataset });
}
});
return acc;
}, []);
const transformRefs = transformPathDatasets.reduce<EsAssetReference[]>(
(acc, transformPathDataset) => {
if (transformPathDataset) {
acc.push({
id: getTransformNameForInstallation(transformPathDataset, installNameSuffix),
type: ElasticsearchAssetType.transform,
});
}
return acc;
},
[]
);
// get and save transform refs before installing transforms
await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, transformRefs);
const transforms: TransformInstallation[] = transformPathDatasets.map(
(transformPathDataset: TransformPathDataset) => {
return {
installationName: getTransformNameForInstallation(
transformPathDataset,
installNameSuffix
),
content: getAsset(transformPathDataset.path).toString('utf-8'),
};
}
);
const installationPromises = transforms.map(async (transform) => {
return installTransform({ callCluster, transform });
});
installedTransforms = await Promise.all(installationPromises).then((results) => results.flat());
}
if (previousInstalledTransformEsAssets.length > 0) {
const currentInstallation = await getInstallation({
savedObjectsClient,
pkgName: registryPackage.name,
});
// remove the saved object reference
await deleteTransformRefs(
savedObjectsClient,
currentInstallation?.installed_es || [],
registryPackage.name,
previousInstalledTransformEsAssets.map((asset) => asset.id),
installedTransforms.map((installed) => installed.id)
);
}
return installedTransforms;
};
const isTransform = (path: string) => {
const pathParts = Registry.pathParts(path);
return pathParts.type === ElasticsearchAssetType.transform;
};
const isDatasetTransform = (path: string, datasetName: string) => {
const pathParts = Registry.pathParts(path);
return (
!path.endsWith('/') &&
pathParts.type === ElasticsearchAssetType.transform &&
pathParts.dataset !== undefined &&
datasetName === pathParts.dataset
);
};
async function installTransform({
callCluster,
transform,
}: {
callCluster: CallESAsCurrentUser;
transform: TransformInstallation;
}): Promise<EsAssetReference> {
// defer validation on put if the source index is not available
await callCluster('transport.request', {
method: 'PUT',
path: `_transform/${transform.installationName}`,
query: 'defer_validation=true',
body: transform.content,
});
await callCluster('transport.request', {
method: 'POST',
path: `_transform/${transform.installationName}/_start`,
});
return { id: transform.installationName, type: ElasticsearchAssetType.transform };
}
const getTransformNameForInstallation = (
transformDataset: TransformPathDataset,
suffix: string
) => {
const filename = transformDataset?.path.split('/')?.pop()?.split('.')[0];
return `${transformDataset.dataset.type}-${transformDataset.dataset.name}-${filename}-${suffix}`;
};

69
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.test.ts Normal file
View file

@ -0,0 +1,69 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsClientContract } from 'kibana/server';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
import { deleteTransformRefs } from './remove';
import { EsAssetReference } from '../../../../../common/types/models';
describe('test transform install', () => {
let savedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
beforeEach(() => {
savedObjectsClient = savedObjectsClientMock.create();
});
test('can delete transform ref and handle duplicate when previous version and current version are the same', async () => {
await deleteTransformRefs(
savedObjectsClient,
[
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
] as EsAssetReference[],
'endpoint',
['metrics-endpoint.metadata-current-default-0.16.0-dev.0'],
['metrics-endpoint.metadata-current-default-0.16.0-dev.0']
);
expect(savedObjectsClient.update.mock.calls).toEqual([
[
'epm-packages',
'endpoint',
{
installed_es: [
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
],
},
],
]);
});
test('can delete transform ref when previous version and current version are not the same', async () => {
await deleteTransformRefs(
savedObjectsClient,
[
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
] as EsAssetReference[],
'endpoint',
['metrics-endpoint.metadata-current-default-0.15.0-dev.0'],
['metrics-endpoint.metadata-current-default-0.16.0-dev.0']
);
expect(savedObjectsClient.update.mock.calls).toEqual([
[
'epm-packages',
'endpoint',
{
installed_es: [
{ id: 'metrics-endpoint.policy-0.16.0-dev.0', type: 'ingest_pipeline' },
{ id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0', type: 'transform' },
],
},
],
]);
});
});

58
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts Normal file
View file

@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { SavedObjectsClientContract } from 'kibana/server';
import { CallESAsCurrentUser, ElasticsearchAssetType, EsAssetReference } from '../../../../types';
import { PACKAGES_SAVED_OBJECT_TYPE } from '../../../../../common/constants';
export const stopTransforms = async (transformIds: string[], callCluster: CallESAsCurrentUser) => {
for (const transformId of transformIds) {
await callCluster('transport.request', {
method: 'POST',
path: `_transform/${transformId}/_stop`,
query: 'force=true',
ignore: [404],
});
}
};
export const deleteTransforms = async (
callCluster: CallESAsCurrentUser,
transformIds: string[]
) => {
await Promise.all(
transformIds.map(async (transformId) => {
await stopTransforms([transformId], callCluster);
await callCluster('transport.request', {
method: 'DELETE',
query: 'force=true',
path: `_transform/${transformId}`,
ignore: [404],
});
})
);
};
export const deleteTransformRefs = async (
savedObjectsClient: SavedObjectsClientContract,
installedEsAssets: EsAssetReference[],
pkgName: string,
installedEsIdToRemove: string[],
currentInstalledEsTransformIds: string[]
) => {
const seen = new Set<string>();
const filteredAssets = installedEsAssets.filter(({ type, id }) => {
if (type !== ElasticsearchAssetType.transform) return true;
const add =
(currentInstalledEsTransformIds.includes(id) || !installedEsIdToRemove.includes(id)) &&
!seen.has(id);
seen.add(id);
return add;
});
return savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
installed_es: filteredAssets,
});
};

420
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts Normal file
View file

@ -0,0 +1,420 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
jest.mock('../../packages/get', () => {
return { getInstallation: jest.fn(), getInstallationObject: jest.fn() };
});
jest.mock('./common', () => {
return {
getAsset: jest.fn(),
};
});
import { installTransformForDataset } from './install';
import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server';
import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types';
import { getInstallation, getInstallationObject } from '../../packages';
import { getAsset } from './common';
// eslint-disable-next-line @kbn/eslint/no-restricted-paths
import { savedObjectsClientMock } from '../../../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock';
describe('test transform install', () => {
let legacyScopedClusterClient: jest.Mocked<ILegacyScopedClusterClient>;
let savedObjectsClient: jest.Mocked<SavedObjectsClientContract>;
beforeEach(() => {
legacyScopedClusterClient = {
callAsInternalUser: jest.fn(),
callAsCurrentUser: jest.fn(),
};
(getInstallation as jest.MockedFunction<typeof getInstallation>).mockReset();
(getInstallationObject as jest.MockedFunction<typeof getInstallationObject>).mockReset();
savedObjectsClient = savedObjectsClientMock.create();
});
afterEach(() => {
jest.clearAllMocks();
});
test('can install new versions and removes older version', async () => {
const previousInstallation: Installation = ({
installed_es: [
{
id: 'metrics-endpoint.policy-0.16.0-dev.0',
type: ElasticsearchAssetType.ingestPipeline,
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
} as unknown) as Installation;
const currentInstallation: Installation = ({
installed_es: [
{
id: 'metrics-endpoint.policy-0.16.0-dev.0',
type: ElasticsearchAssetType.ingestPipeline,
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
} as unknown) as Installation;
(getAsset as jest.MockedFunction<typeof getAsset>)
.mockReturnValueOnce(Buffer.from('{"content": "data"}', 'utf8'))
.mockReturnValueOnce(Buffer.from('{"content": "data"}', 'utf8'));
(getInstallation as jest.MockedFunction<typeof getInstallation>)
.mockReturnValueOnce(Promise.resolve(previousInstallation))
.mockReturnValueOnce(Promise.resolve(currentInstallation));
(getInstallationObject as jest.MockedFunction<
typeof getInstallationObject
>).mockReturnValueOnce(
Promise.resolve(({
attributes: {
installed_es: previousInstallation.installed_es,
},
} as unknown) as SavedObject<Installation>)
);
await installTransformForDataset(
({
name: 'endpoint',
version: '0.16.0-dev.0',
datasets: [
{
type: 'metrics',
name: 'endpoint.metadata',
title: 'Endpoint Metadata',
release: 'experimental',
package: 'endpoint',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
dynamic: false,
},
},
path: 'metadata',
},
{
type: 'metrics',
name: 'endpoint.metadata_current',
title: 'Endpoint Metadata Current',
release: 'experimental',
package: 'endpoint',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
dynamic: false,
},
},
path: 'metadata_current',
},
],
} as unknown) as RegistryPackage,
[
'endpoint-0.16.0-dev.0/dataset/policy/elasticsearch/ingest_pipeline/default.json',
'endpoint-0.16.0-dev.0/dataset/metadata/elasticsearch/transform/default.json',
'endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json',
],
legacyScopedClusterClient.callAsCurrentUser,
savedObjectsClient
);
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
[
'transport.request',
{
method: 'POST',
path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0/_stop',
query: 'force=true',
ignore: [404],
},
],
[
'transport.request',
{
method: 'DELETE',
query: 'force=true',
path: '_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0',
ignore: [404],
},
],
[
'transport.request',
{
method: 'PUT',
path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
],
[
'transport.request',
{
method: 'PUT',
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
],
[
'transport.request',
{
method: 'POST',
path: '_transform/metrics-endpoint.metadata-default-0.16.0-dev.0/_start',
},
],
[
'transport.request',
{
method: 'POST',
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
},
],
]);
expect(savedObjectsClient.update.mock.calls).toEqual([
[
'epm-packages',
'endpoint',
{
installed_es: [
{
id: 'metrics-endpoint.policy-0.16.0-dev.0',
type: 'ingest_pipeline',
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
type: 'transform',
},
],
},
],
[
'epm-packages',
'endpoint',
{
installed_es: [
{
id: 'metrics-endpoint.policy-0.16.0-dev.0',
type: 'ingest_pipeline',
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
type: 'transform',
},
],
},
],
]);
});
test('can install new version and when no older version', async () => {
const previousInstallation: Installation = ({
installed_es: [],
} as unknown) as Installation;
const currentInstallation: Installation = ({
installed_es: [
{
id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
} as unknown) as Installation;
(getAsset as jest.MockedFunction<typeof getAsset>).mockReturnValueOnce(
Buffer.from('{"content": "data"}', 'utf8')
);
(getInstallation as jest.MockedFunction<typeof getInstallation>)
.mockReturnValueOnce(Promise.resolve(previousInstallation))
.mockReturnValueOnce(Promise.resolve(currentInstallation));
(getInstallationObject as jest.MockedFunction<
typeof getInstallationObject
>).mockReturnValueOnce(
Promise.resolve(({ attributes: { installed_es: [] } } as unknown) as SavedObject<
Installation
>)
);
legacyScopedClusterClient.callAsCurrentUser = jest.fn();
await installTransformForDataset(
({
name: 'endpoint',
version: '0.16.0-dev.0',
datasets: [
{
type: 'metrics',
name: 'endpoint.metadata_current',
title: 'Endpoint Metadata',
release: 'experimental',
package: 'endpoint',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
dynamic: false,
},
},
path: 'metadata_current',
},
],
} as unknown) as RegistryPackage,
['endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json'],
legacyScopedClusterClient.callAsCurrentUser,
savedObjectsClient
);
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
[
'transport.request',
{
method: 'PUT',
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
],
[
'transport.request',
{
method: 'POST',
path: '_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
},
],
]);
expect(savedObjectsClient.update.mock.calls).toEqual([
[
'epm-packages',
'endpoint',
{
installed_es: [
{ id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' },
],
},
],
]);
});
test('can removes older version when no new install in package', async () => {
const previousInstallation: Installation = ({
installed_es: [
{
id: 'metrics-endpoint.metadata-current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
} as unknown) as Installation;
const currentInstallation: Installation = ({
installed_es: [],
} as unknown) as Installation;
(getInstallation as jest.MockedFunction<typeof getInstallation>)
.mockReturnValueOnce(Promise.resolve(previousInstallation))
.mockReturnValueOnce(Promise.resolve(currentInstallation));
(getInstallationObject as jest.MockedFunction<
typeof getInstallationObject
>).mockReturnValueOnce(
Promise.resolve(({
attributes: { installed_es: currentInstallation.installed_es },
} as unknown) as SavedObject<Installation>)
);
await installTransformForDataset(
({
name: 'endpoint',
version: '0.16.0-dev.0',
datasets: [
{
type: 'metrics',
name: 'endpoint.metadata',
title: 'Endpoint Metadata',
release: 'experimental',
package: 'endpoint',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
dynamic: false,
},
},
path: 'metadata',
},
{
type: 'metrics',
name: 'endpoint.metadata_current',
title: 'Endpoint Metadata Current',
release: 'experimental',
package: 'endpoint',
ingest_pipeline: 'default',
elasticsearch: {
'index_template.mappings': {
dynamic: false,
},
},
path: 'metadata_current',
},
],
} as unknown) as RegistryPackage,
[],
legacyScopedClusterClient.callAsCurrentUser,
savedObjectsClient
);
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
[
'transport.request',
{
ignore: [404],
method: 'POST',
path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0/_stop',
query: 'force=true',
},
],
[
'transport.request',
{
ignore: [404],
method: 'DELETE',
path: '_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0',
query: 'force=true',
},
],
]);
expect(savedObjectsClient.update.mock.calls).toEqual([
[
'epm-packages',
'endpoint',
{
installed_es: [],
},
],
]);
});
});

16
x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
View file

@ -35,6 +35,7 @@ import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
import { deleteKibanaSavedObjectsAssets } from './remove';
import { PackageOutdatedError } from '../../../errors';
import { getPackageSavedObjects } from './get';
import { installTransformForDataset } from '../elasticsearch/transform/install';
export async function installLatestPackage(options: {
savedObjectsClient: SavedObjectsClientContract;
@ -191,6 +192,13 @@ export async function installPackage({
// update current backing indices of each data stream
await updateCurrentWriteIndices(callCluster, installedTemplates);
const installedTransforms = await installTransformForDataset(
registryPackageInfo,
paths,
callCluster,
savedObjectsClient
);
// if this is an update or retrying an update, delete the previous version's pipelines
if (installType === 'update' || installType === 'reupdate') {
await deletePreviousPipelines(
@ -216,13 +224,19 @@ export async function installPackage({
type: ElasticsearchAssetType.indexTemplate,
}));
await Promise.all([installKibanaAssetsPromise, installIndexPatternPromise]);
// update to newly installed version when all assets are successfully installed
if (installedPkg) await updateVersion(savedObjectsClient, pkgName, pkgVersion);
await savedObjectsClient.update(PACKAGES_SAVED_OBJECT_TYPE, pkgName, {
install_version: pkgVersion,
install_status: 'installed',
});
return [...installedKibanaAssetsRefs, ...installedPipelines, ...installedTemplateRefs];
return [
...installedKibanaAssetsRefs,
...installedPipelines,
...installedTemplateRefs,
...installedTransforms,
];
}
const updateVersion = async (

13
x-pack/plugins/ingest_manager/server/services/epm/packages/remove.ts
View file

@ -6,12 +6,17 @@
import { SavedObjectsClientContract } from 'src/core/server';
import Boom from 'boom';
import { PACKAGES_SAVED_OBJECT_TYPE, PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../../constants';
import { AssetReference, AssetType, ElasticsearchAssetType } from '../../../types';
import { CallESAsCurrentUser } from '../../../types';
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE, PACKAGES_SAVED_OBJECT_TYPE } from '../../../constants';
import {
AssetReference,
AssetType,
CallESAsCurrentUser,
ElasticsearchAssetType,
} from '../../../types';
import { getInstallation, savedObjectTypes } from './index';
import { deletePipeline } from '../elasticsearch/ingest_pipeline/';
import { installIndexPatterns } from '../kibana/index_pattern/install';
import { deleteTransforms } from '../elasticsearch/transform/remove';
import { packagePolicyService, appContextService } from '../..';
import { splitPkgKey, deletePackageCache, getArchiveInfo } from '../registry';
@ -72,6 +77,8 @@ async function deleteAssets(
return deletePipeline(callCluster, id);
} else if (assetType === ElasticsearchAssetType.indexTemplate) {
return deleteTemplate(callCluster, id);
} else if (assetType === ElasticsearchAssetType.transform) {
return deleteTransforms(callCluster, [id]);
}
});
try {

1
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -7,6 +7,7 @@
export const eventsIndexPattern = 'logs-endpoint.events.*';
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current-*';
export const policyIndexPattern = 'metrics-endpoint.policy-*';
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

7
x-pack/plugins/security_solution/common/endpoint/types/index.ts
View file

@ -445,6 +445,13 @@ export type HostInfo = Immutable<{
host_status: HostStatus;
}>;
export type HostMetadataDetails = Immutable<{
agent: {
id: string;
};
HostDetails: HostMetadata;
}>;
export type HostMetadata = Immutable<{
'@timestamp': number;
event: {

29
x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts
View file

@ -9,11 +9,12 @@ import { SearchResponse } from 'elasticsearch';
import { schema } from '@kbn/config-schema';
import Boom from 'boom';
import { metadataIndexPattern } from '../../../../common/endpoint/constants';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
import { getESQueryHostMetadataByID, kibanaRequestToMetadataListESQuery } from './query_builders';
import {
HostInfo,
HostMetadata,
HostMetadataDetails,
HostResultList,
HostStatus,
} from '../../../../common/endpoint/types';
@ -23,10 +24,6 @@ import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/m
import { findAllUnenrolledAgentIds } from './support/unenroll';
import { findAgentIDsByStatus } from './support/agent_status';
interface HitSource {
_source: HostMetadata;
}
interface MetadataRequestContext {
agentService: AgentService;
logger: Logger;
@ -127,7 +124,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
const queryParams = await kibanaRequestToMetadataListESQuery(
req,
endpointAppContext,
metadataIndexPattern,
metadataCurrentIndexPattern,
{
unenrolledAgentIds: unenrolledAgentIds.concat(IGNORED_ELASTIC_AGENT_IDS),
statusAgentIDs: statusIDs,
@ -137,7 +134,7 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp
const response = (await context.core.elasticsearch.legacy.client.callAsCurrentUser(
'search',
queryParams
)) as SearchResponse<HostMetadata>;
)) as SearchResponse<HostMetadataDetails>;
return res.ok({
body: await mapToHostResultList(queryParams, response, metadataRequestContext),
@ -193,17 +190,17 @@ export async function getHostData(
metadataRequestContext: MetadataRequestContext,
id: string
): Promise<HostInfo | undefined> {
const query = getESQueryHostMetadataByID(id, metadataIndexPattern);
const query = getESQueryHostMetadataByID(id, metadataCurrentIndexPattern);
const response = (await metadataRequestContext.requestHandlerContext.core.elasticsearch.legacy.client.callAsCurrentUser(
'search',
query
)) as SearchResponse<HostMetadata>;
)) as SearchResponse<HostMetadataDetails>;
if (response.hits.hits.length === 0) {
return undefined;
}
const hostMetadata: HostMetadata = response.hits.hits[0]._source;
const hostMetadata: HostMetadata = response.hits.hits[0]._source.HostDetails;
const agent = await findAgent(metadataRequestContext, hostMetadata);
if (agent && !agent.active) {
@ -241,19 +238,19 @@ async function findAgent(
async function mapToHostResultList(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
queryParams: Record<string, any>,
searchResponse: SearchResponse<HostMetadata>,
searchResponse: SearchResponse<HostMetadataDetails>,
metadataRequestContext: MetadataRequestContext
): Promise<HostResultList> {
const totalNumberOfHosts = searchResponse?.aggregations?.total?.value || 0;
const totalNumberOfHosts =
((searchResponse.hits?.total as unknown) as { value: number; relation: string }).value || 0;
if (searchResponse.hits.hits.length > 0) {
return {
request_page_size: queryParams.size,
request_page_index: queryParams.from,
hosts: await Promise.all(
searchResponse.hits.hits
.map((response) => response.inner_hits.most_recent.hits.hits)
.flatMap((data) => data as HitSource)
.map(async (entry) => enrichHostMetadata(entry._source, metadataRequestContext))
searchResponse.hits.hits.map(async (entry) =>
enrichHostMetadata(entry._source.HostDetails, metadataRequestContext)
)
),
total: totalNumberOfHosts,
};

43
x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts
View file

@ -23,6 +23,7 @@ import {
import {
HostInfo,
HostMetadata,
HostMetadataDetails,
HostResultList,
HostStatus,
} from '../../../../common/endpoint/types';
@ -141,7 +142,7 @@ describe('test endpoint route', () => {
bool: {
must_not: {
terms: {
'elastic.agent.id': [
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
@ -197,7 +198,7 @@ describe('test endpoint route', () => {
bool: {
must_not: {
terms: {
'elastic.agent.id': [
'HostDetails.elastic.agent.id': [
'00000000-0000-0000-0000-000000000000',
'11111111-1111-1111-1111-111111111111',
],
@ -442,7 +443,7 @@ describe('Filters Schema Test', () => {
});
});
function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadata> {
function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostMetadataDetails> {
return ({
took: 15,
timed_out: false,
@ -454,7 +455,7 @@ function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostM
},
hits: {
total: {
value: 5,
value: 1,
relation: 'eq',
},
max_score: null,
@ -464,36 +465,18 @@ function createSearchResponse(hostMetadata?: HostMetadata): SearchResponse<HostM
_index: 'metrics-endpoint.metadata-default',
_id: '8FhM0HEBYyRTvb6lOQnw',
_score: null,
_source: hostMetadata,
sort: [1588337587997],
inner_hits: {
most_recent: {
hits: {
total: {
value: 2,
relation: 'eq',
},
max_score: null,
hits: [
{
_index: 'metrics-endpoint.metadata-default',
_id: 'W6Vo1G8BYQH1gtPUgYkC',
_score: null,
_source: hostMetadata,
sort: [1579816615336],
},
],
},
_source: {
agent: {
id: '1e3472bb-5c20-4946-b469-b5af1a809e4f',
},
HostDetails: {
...hostMetadata,
},
},
sort: [1588337587997],
},
]
: [],
},
aggregations: {
total: {
value: 1,
},
},
} as unknown) as SearchResponse<HostMetadata>;
} as unknown) as SearchResponse<HostMetadataDetails>;
}

106
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.test.ts
View file

@ -7,7 +7,7 @@ import { httpServerMock, loggingSystemMock } from '../../../../../../../src/core
import { kibanaRequestToMetadataListESQuery, getESQueryHostMetadataByID } from './query_builders';
import { EndpointAppContextService } from '../../endpoint_app_context_services';
import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__';
import { metadataIndexPattern } from '../../../../common/endpoint/constants';
import { metadataCurrentIndexPattern } from '../../../../common/endpoint/constants';
describe('query builder', () => {
describe('MetadataListESQuery', () => {
@ -22,31 +22,16 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataIndexPattern
metadataCurrentIndexPattern
);
expect(query).toEqual({
body: {
query: {
match_all: {},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},
@ -54,7 +39,7 @@ describe('query builder', () => {
},
from: 0,
size: 10,
index: metadataIndexPattern,
index: metadataCurrentIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
});
@ -74,7 +59,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataIndexPattern,
metadataCurrentIndexPattern,
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
@ -86,29 +71,14 @@ describe('query builder', () => {
bool: {
must_not: {
terms: {
'elastic.agent.id': [unenrolledElasticAgentId],
'HostDetails.elastic.agent.id': [unenrolledElasticAgentId],
},
},
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},
@ -116,7 +86,7 @@ describe('query builder', () => {
},
from: 0,
size: 10,
index: metadataIndexPattern,
index: metadataCurrentIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
}
@ -127,7 +97,7 @@ describe('query builder', () => {
it('test default query params for all endpoints metadata when body filter is provided', async () => {
const mockRequest = httpServerMock.createKibanaRequest({
body: {
filters: { kql: 'not host.ip:10.140.73.246' },
filters: { kql: 'not HostDetails.host.ip:10.140.73.246' },
},
});
const query = await kibanaRequestToMetadataListESQuery(
@ -137,7 +107,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataIndexPattern
metadataCurrentIndexPattern
);
expect(query).toEqual({
@ -152,7 +122,7 @@ describe('query builder', () => {
should: [
{
match: {
'host.ip': '10.140.73.246',
'HostDetails.host.ip': '10.140.73.246',
},
},
],
@ -164,24 +134,9 @@ describe('query builder', () => {
],
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},
@ -189,7 +144,7 @@ describe('query builder', () => {
},
from: 0,
size: 10,
index: metadataIndexPattern,
index: metadataCurrentIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
});
@ -201,7 +156,7 @@ describe('query builder', () => {
const unenrolledElasticAgentId = '1fdca33f-799f-49f4-939c-ea4383c77672';
const mockRequest = httpServerMock.createKibanaRequest({
body: {
filters: { kql: 'not host.ip:10.140.73.246' },
filters: { kql: 'not HostDetails.host.ip:10.140.73.246' },
},
});
const query = await kibanaRequestToMetadataListESQuery(
@ -211,7 +166,7 @@ describe('query builder', () => {
service: new EndpointAppContextService(),
config: () => Promise.resolve(createMockConfig()),
},
metadataIndexPattern,
metadataCurrentIndexPattern,
{
unenrolledAgentIds: [unenrolledElasticAgentId],
}
@ -226,7 +181,7 @@ describe('query builder', () => {
bool: {
must_not: {
terms: {
'elastic.agent.id': [unenrolledElasticAgentId],
'HostDetails.elastic.agent.id': [unenrolledElasticAgentId],
},
},
},
@ -238,7 +193,7 @@ describe('query builder', () => {
should: [
{
match: {
'host.ip': '10.140.73.246',
'HostDetails.host.ip': '10.140.73.246',
},
},
],
@ -250,24 +205,9 @@ describe('query builder', () => {
],
},
},
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},
@ -275,7 +215,7 @@ describe('query builder', () => {
},
from: 0,
size: 10,
index: metadataIndexPattern,
index: metadataCurrentIndexPattern,
// eslint-disable-next-line @typescript-eslint/no-explicit-any
} as Record<string, any>);
}
@ -285,15 +225,15 @@ describe('query builder', () => {
describe('MetadataGetQuery', () => {
it('searches for the correct ID', () => {
const mockID = 'AABBCCDD-0011-2233-AA44-DEADBEEF8899';
const query = getESQueryHostMetadataByID(mockID, metadataIndexPattern);
const query = getESQueryHostMetadataByID(mockID, metadataCurrentIndexPattern);
expect(query).toEqual({
body: {
query: { match: { 'host.id': mockID } },
sort: [{ 'event.created': { order: 'desc' } }],
query: { match: { 'HostDetails.host.id': mockID } },
sort: [{ 'HostDetails.event.created': { order: 'desc' } }],
size: 1,
},
index: metadataIndexPattern,
index: metadataCurrentIndexPattern,
});
});
});

25
x-pack/plugins/security_solution/server/endpoint/routes/metadata/query_builders.ts
View file

@ -28,24 +28,9 @@ export async function kibanaRequestToMetadataListESQuery(
queryBuilderOptions?.unenrolledAgentIds!,
queryBuilderOptions?.statusAgentIDs!
),
collapse: {
field: 'host.id',
inner_hits: {
name: 'most_recent',
size: 1,
sort: [{ 'event.created': 'desc' }],
},
},
aggs: {
total: {
cardinality: {
field: 'host.id',
},
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},
@ -90,7 +75,7 @@ function buildQueryBody(
? {
must_not: {
terms: {
'elastic.agent.id': unerolledAgentIds,
'HostDetails.elastic.agent.id': unerolledAgentIds,
},
},
}
@ -99,7 +84,7 @@ function buildQueryBody(
? {
must: {
terms: {
'elastic.agent.id': statusAgentIDs,
'HostDetails.elastic.agent.id': statusAgentIDs,
},
},
}
@ -137,12 +122,12 @@ export function getESQueryHostMetadataByID(hostID: string, index: string) {
body: {
query: {
match: {
'host.id': hostID,
'HostDetails.host.id': hostID,
},
},
sort: [
{
'event.created': {
'HostDetails.event.created': {
order: 'desc',
},
},

23
x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts
View file

@ -84,6 +84,13 @@ export default function (providerContext: FtrProviderContext) {
});
expect(resSettings.statusCode).equal(200);
});
it('should have installed the transform components', async function () {
const res = await es.transport.request({
method: 'GET',
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
});
expect(res.statusCode).equal(200);
});
it('should have installed the kibana assets', async function () {
const resIndexPatternLogs = await kibanaServer.savedObjects.get({
type: 'index-pattern',
@ -161,6 +168,10 @@ export default function (providerContext: FtrProviderContext) {
id: 'metrics-all_assets.test_metrics',
type: 'index_template',
},
{
id: 'logs-all_assets.test_logs-default-0.1.0',
type: 'transform',
},
],
es_index_patterns: {
test_logs: 'logs-all_assets.test_logs-*',
@ -237,6 +248,18 @@ export default function (providerContext: FtrProviderContext) {
);
expect(resPipeline2.statusCode).equal(404);
});
it('should have uninstalled the transforms', async function () {
const res = await es.transport.request(
{
method: 'GET',
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
},
{
ignore: [404],
}
);
expect(res.statusCode).equal(404);
});
it('should have uninstalled the kibana assets', async function () {
let resDashboard;
try {

35
x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/dataset/test_logs/elasticsearch/transform/default.json Normal file
View file

@ -0,0 +1,35 @@
{
"source": {
"index": "logs-all_assets.test_log-default*"
},
"dest": {
"index": "logs-all_assets.test_log_current-default"
},
"pivot": {
"group_by": {
"agent.id": {
"terms": {
"field": "agent.id"
}
}
},
"aggregations": {
"HostDetails": {
"scripted_metric": {
"init_script": "state.timestamp_latest = 0L; state.last_doc=''",
"map_script": "def current_date = doc['@timestamp'].getValue().toInstant().toEpochMilli(); if (current_date \u003e state.timestamp_latest) {state.timestamp_latest = current_date;state.last_doc = new HashMap(params['_source']);}",
"combine_script": "return state",
"reduce_script": "def last_doc = '';def timestamp_latest = 0L; for (s in states) {if (s.timestamp_latest \u003e (timestamp_latest)) {timestamp_latest = s.timestamp_latest; last_doc = s.last_doc;}} return last_doc"
}
}
}
},
"description": "collapse and update the latest document for each host",
"frequency": "1m",
"sync": {
"time": {
"field": "event.ingested",
"delay": "60s"
}
}
}

2
x-pack/test/ingest_manager_api_integration/config.ts
View file

@ -12,7 +12,7 @@ import { defineDockerServersConfig } from '@kbn/test';
// Docker image to use for Ingest Manager API integration tests.
// This hash comes from the commit hash here: https://github.com/elastic/package-storage/commit
export const dockerImage =
'docker.elastic.co/package-registry/distribution:f6b01daec8cfe355101e366de9941d35a4c3763e';
'docker.elastic.co/package-registry/distribution:5e0e12ce1bc2cb0c2f67f2e07d11b9a6043bcf25';
export default async function ({ readConfigFile }: FtrConfigProviderContext) {
const xPackAPITestsConfig = await readConfigFile(require.resolve('../api_integration/config.ts'));

37
x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts
View file

@ -6,7 +6,12 @@
import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import { deleteMetadataStream } from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
import {
deleteMetadataCurrentStream,
deleteMetadataStream,
} from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
export default ({ getPageObjects, getService }: FtrProviderContext) => {
const pageObjects = getPageObjects(['common', 'endpoint', 'header', 'endpointPageUtils']);
const esArchiver = getService('esArchiver');
@ -23,6 +28,16 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
'Version',
'Last Active',
],
[
'rezzani-7.example.com',
'Error',
'Default',
'Failure',
'windows 10.0',
'10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c',
'6.8.0',
'Jan 24, 2020 @ 16:06:09.541',
],
[
'cadmann-4.example.com',
'Error',
@ -43,16 +58,6 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
'6.0.0',
'Jan 24, 2020 @ 16:06:09.541',
],
[
'rezzani-7.example.com',
'Error',
'Default',
'Failure',
'windows 10.0',
'10.101.149.26, 2606:a000:ffc0:39:11ef:37b9:3371:578c',
'6.8.0',
'Jan 24, 2020 @ 16:06:09.541',
],
];
describe('endpoint list', function () {
@ -61,10 +66,13 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
describe('when initially navigating to page', () => {
before(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await pageObjects.endpoint.navigateToEndpointList();
});
after(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
});
it('finds no data in list and prompts onboarding to add policy', async () => {
@ -73,7 +81,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
it('finds data after load and polling', async () => {
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 10000);
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 120000);
const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable');
expect(tableData).to.eql(expectedData);
});
@ -82,10 +90,12 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
describe('when there is data,', () => {
before(async () => {
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
await sleep(120000);
await pageObjects.endpoint.navigateToEndpointList();
});
after(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
});
it('finds page title', async () => {
@ -202,10 +212,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
});
});
describe('when there is no data,', () => {
describe.skip('when there is no data,', () => {
before(async () => {
// clear out the data and reload the page
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await pageObjects.endpoint.navigateToEndpointList();
});
it('displays empty Policy Table page.', async () => {

35
x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts
View file

@ -10,6 +10,7 @@ import {
eventsIndexPattern,
alertsIndexPattern,
policyIndexPattern,
metadataCurrentIndexPattern,
} from '../../../plugins/security_solution/common/endpoint/constants';
export async function deleteDataStream(getService: (serviceName: 'es') => Client, index: string) {
@ -25,10 +26,44 @@ export async function deleteDataStream(getService: (serviceName: 'es') => Client
);
}
export async function deleteAllDocsFromIndex(
getService: (serviceName: 'es') => Client,
index: string
) {
const client = getService('es');
await client.deleteByQuery(
{
body: {
query: {
match_all: {},
},
},
index: `${index}`,
},
{
ignore: [404],
}
);
}
export async function deleteMetadataStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, metadataIndexPattern);
}
export async function deleteMetadataCurrentStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, metadataCurrentIndexPattern);
}
export async function deleteAllDocsFromMetadataIndex(getService: (serviceName: 'es') => Client) {
await deleteAllDocsFromIndex(getService, metadataIndexPattern);
}
export async function deleteAllDocsFromMetadataCurrentIndex(
getService: (serviceName: 'es') => Client
) {
await deleteAllDocsFromIndex(getService, metadataCurrentIndexPattern);
}
export async function deleteEventsStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, eventsIndexPattern);
}

54
x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts
View file

@ -5,7 +5,12 @@
*/
import expect from '@kbn/expect/expect.js';
import { FtrProviderContext } from '../ftr_provider_context';
import { deleteMetadataStream } from './data_stream_helper';
import {
deleteAllDocsFromMetadataCurrentIndex,
deleteMetadataCurrentStream,
deleteAllDocsFromMetadataIndex,
deleteMetadataStream,
} from './data_stream_helper';
/**
* The number of host documents in the es archive.
@ -15,12 +20,14 @@ const numberOfHostsInFixture = 3;
export default function ({ getService }: FtrProviderContext) {
const esArchiver = getService('esArchiver');
const supertest = getService('supertest');
describe('test metadata api', () => {
describe('POST /api/endpoint/metadata when index is empty', () => {
it('metadata api should return empty result when index is empty', async () => {
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
// to do it manually
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
const { body } = await supertest
.post('/api/endpoint/metadata')
.set('kbn-xsrf', 'xxx')
@ -34,12 +41,19 @@ export default function ({ getService }: FtrProviderContext) {
});
describe('POST /api/endpoint/metadata when index is not empty', () => {
before(
async () => await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true })
);
before(async () => {
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
// wait for transform
await new Promise((r) => setTimeout(r, 120000));
});
// the endpoint uses data streams and es archiver does not support deleting them at the moment so we need
// to do it manually
after(async () => await deleteMetadataStream(getService));
after(async () => {
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
});
it('metadata api should return one entry for each host with default paging', async () => {
const { body } = await supertest
.post('/api/endpoint/metadata')
@ -121,7 +135,7 @@ export default function ({ getService }: FtrProviderContext) {
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: 'not host.ip:10.46.229.234',
kql: 'not HostDetails.host.ip:10.46.229.234',
},
})
.expect(200);
@ -146,7 +160,7 @@ export default function ({ getService }: FtrProviderContext) {
},
],
filters: {
kql: `not host.ip:${notIncludedIp}`,
kql: `not HostDetails.host.ip:${notIncludedIp}`,
},
})
.expect(200);
@ -154,12 +168,14 @@ export default function ({ getService }: FtrProviderContext) {
const resultIps: string[] = [].concat(
...body.hosts.map((hostInfo: Record<string, any>) => hostInfo.metadata.host.ip)
);
expect(resultIps).to.eql([
'10.192.213.130',
'10.70.28.129',
'10.101.149.26',
'2606:a000:ffc0:39:11ef:37b9:3371:578c',
]);
expect(resultIps.sort()).to.eql(
[
'10.192.213.130',
'10.70.28.129',
'10.101.149.26',
'2606:a000:ffc0:39:11ef:37b9:3371:578c',
].sort()
);
expect(resultIps).not.include.eql(notIncludedIp);
expect(body.hosts.length).to.eql(2);
expect(body.request_page_size).to.eql(10);
@ -173,7 +189,7 @@ export default function ({ getService }: FtrProviderContext) {
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `host.os.Ext.variant:${variantValue}`,
kql: `HostDetails.host.os.Ext.variant:${variantValue}`,
},
})
.expect(200);
@ -194,7 +210,7 @@ export default function ({ getService }: FtrProviderContext) {
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `host.ip:${targetEndpointIp}`,
kql: `HostDetails.host.ip:${targetEndpointIp}`,
},
})
.expect(200);
@ -215,7 +231,7 @@ export default function ({ getService }: FtrProviderContext) {
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `not Endpoint.policy.applied.status:success`,
kql: `not HostDetails.Endpoint.policy.applied.status:success`,
},
})
.expect(200);
@ -236,7 +252,7 @@ export default function ({ getService }: FtrProviderContext) {
.set('kbn-xsrf', 'xxx')
.send({
filters: {
kql: `elastic.agent.id:${targetElasticAgentId}`,
kql: `HostDetails.elastic.agent.id:${targetElasticAgentId}`,
},
})
.expect(200);