maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

78024: move transform out of dataset (#78216)

Browse source 
* 78024: move transform out of dataset

* Change index prefix

* 78024: fix tests, remove vestiges

* 78024: remove index defined in the transform when transform is removed.

* 78024: clean up

* 78024: fix build

* 78024: add comment

* 78024: remove test I added

* 78024: more removal, will add in next PR

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
This commit is contained in:
nnamdifrankie 2020-09-29 10:30:04 -04:00 committed by GitHub
parent f73bc48b3b
commit 9e9a48be62
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 154 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/install.ts
View file

@ -9,7 +9,6 @@ import { SavedObjectsClientContract } from 'kibana/server';
import { saveInstalledEsRefs } from '../../packages/install';
import * as Registry from '../../registry';
import {
Dataset,
ElasticsearchAssetType,
EsAssetReference,
RegistryPackage,
@ -24,12 +23,7 @@ interface TransformInstallation {
content: string;
}
interface TransformPathDataset {
path: string;
dataset: Dataset;
}
export const installTransformForDataset = async (
export const installTransform = async (
registryPackage: RegistryPackage,
paths: string[],
callCluster: CallESAsCurrentUser,
@ -51,53 +45,32 @@ export const installTransformForDataset = async (
callCluster,
previousInstalledTransformEsAssets.map((asset) => asset.id)
);
// install the latest dataset
const datasets = registryPackage.datasets;
if (!datasets?.length) return [];
const installNameSuffix = `${registryPackage.version}`;
const installNameSuffix = `${registryPackage.version}`;
const transformPaths = paths.filter((path) => isTransform(path));
let installedTransforms: EsAssetReference[] = [];
if (transformPaths.length > 0) {
const transformPathDatasets = datasets.reduce<TransformPathDataset[]>((acc, dataset) => {
transformPaths.forEach((path) => {
if (isDatasetTransform(path, dataset.path)) {
acc.push({ path, dataset });
}
const transformRefs = transformPaths.reduce<EsAssetReference[]>((acc, path) => {
acc.push({
id: getTransformNameForInstallation(registryPackage, path, installNameSuffix),
type: ElasticsearchAssetType.transform,
});
return acc;
}, []);
const transformRefs = transformPathDatasets.reduce<EsAssetReference[]>(
(acc, transformPathDataset) => {
if (transformPathDataset) {
acc.push({
id: getTransformNameForInstallation(transformPathDataset, installNameSuffix),
type: ElasticsearchAssetType.transform,
});
}
return acc;
},
[]
);
// get and save transform refs before installing transforms
await saveInstalledEsRefs(savedObjectsClient, registryPackage.name, transformRefs);
const transforms: TransformInstallation[] = transformPathDatasets.map(
(transformPathDataset: TransformPathDataset) => {
return {
installationName: getTransformNameForInstallation(
transformPathDataset,
installNameSuffix
),
content: getAsset(transformPathDataset.path).toString('utf-8'),
};
}
);
const transforms: TransformInstallation[] = transformPaths.map((path: string) => {
return {
installationName: getTransformNameForInstallation(registryPackage, path, installNameSuffix),
content: getAsset(path).toString('utf-8'),
};
});
const installationPromises = transforms.map(async (transform) => {
return installTransform({ callCluster, transform });
return handleTransformInstall({ callCluster, transform });
});
installedTransforms = await Promise.all(installationPromises).then((results) => results.flat());
@ -123,20 +96,10 @@ export const installTransformForDataset = async (
const isTransform = (path: string) => {
const pathParts = Registry.pathParts(path);
return pathParts.type === ElasticsearchAssetType.transform;
return !path.endsWith('/') && pathParts.type === ElasticsearchAssetType.transform;
};
const isDatasetTransform = (path: string, datasetName: string) => {
const pathParts = Registry.pathParts(path);
return (
!path.endsWith('/') &&
pathParts.type === ElasticsearchAssetType.transform &&
pathParts.dataset !== undefined &&
datasetName === pathParts.dataset
);
};
async function installTransform({
async function handleTransformInstall({
callCluster,
transform,
}: {
@ -160,9 +123,12 @@ async function installTransform({
}
const getTransformNameForInstallation = (
transformDataset: TransformPathDataset,
registryPackage: RegistryPackage,
path: string,
suffix: string
) => {
const filename = transformDataset?.path.split('/')?.pop()?.split('.')[0];
return `${transformDataset.dataset.type}-${transformDataset.dataset.name}-${filename}-${suffix}`;
const pathPaths = path.split('/');
const filename = pathPaths?.pop()?.split('.')[0];
const folderName = pathPaths?.pop();
return `${registryPackage.name}.${folderName}-${filename}-${suffix}`;
};

22
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/remove.ts
View file

@ -25,6 +25,19 @@ export const deleteTransforms = async (
) => {
await Promise.all(
transformIds.map(async (transformId) => {
// get the index the transform
const transformResponse: {
count: number;
transforms: Array<{
dest: {
index: string;
};
}>;
} = await callCluster('transport.request', {
method: 'GET',
path: `/_transform/${transformId}`,
});
await stopTransforms([transformId], callCluster);
await callCluster('transport.request', {
method: 'DELETE',
@ -32,6 +45,15 @@ export const deleteTransforms = async (
path: `/_transform/${transformId}`,
ignore: [404],
});
// expect this to be 1
for (const transform of transformResponse.transforms) {
await callCluster('transport.request', {
method: 'DELETE',
path: `/${transform?.dest?.index}`,
ignore: [404],
});
}
})
);
};

125
x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/transform/transform.test.ts
View file

@ -14,7 +14,7 @@ jest.mock('./common', () => {
};
});
import { installTransformForDataset } from './install';
import { installTransform } from './install';
import { ILegacyScopedClusterClient, SavedObject, SavedObjectsClientContract } from 'kibana/server';
import { ElasticsearchAssetType, Installation, RegistryPackage } from '../../../../types';
import { getInstallation, getInstallationObject } from '../../packages';
@ -47,7 +47,7 @@ describe('test transform install', () => {
type: ElasticsearchAssetType.ingestPipeline,
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
id: 'endpoint.metadata_current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
@ -60,15 +60,15 @@ describe('test transform install', () => {
type: ElasticsearchAssetType.ingestPipeline,
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
id: 'endpoint.metadata_current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
id: 'endpoint.metadata_current-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
id: 'endpoint.metadata-default-0.16.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
@ -91,7 +91,26 @@ describe('test transform install', () => {
} as unknown) as SavedObject<Installation>)
);
await installTransformForDataset(
legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce(
Promise.resolve({
count: 1,
transforms: [
{
dest: {
index: 'index',
},
},
],
} as {
count: number;
transforms: Array<{
dest: {
index: string;
};
}>;
})
);
await installTransform(
({
name: 'endpoint',
version: '0.16.0-dev.0',
@ -128,18 +147,26 @@ describe('test transform install', () => {
} as unknown) as RegistryPackage,
[
'endpoint-0.16.0-dev.0/dataset/policy/elasticsearch/ingest_pipeline/default.json',
'endpoint-0.16.0-dev.0/dataset/metadata/elasticsearch/transform/default.json',
'endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata/default.json',
'endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json',
],
legacyScopedClusterClient.callAsCurrentUser,
savedObjectsClient
);
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
[
'transport.request',
{
method: 'GET',
path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0',
},
],
[
'transport.request',
{
method: 'POST',
path: '/_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0/_stop',
path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0/_stop',
query: 'force=true',
ignore: [404],
},
@ -149,7 +176,15 @@ describe('test transform install', () => {
{
method: 'DELETE',
query: 'force=true',
path: '/_transform/metrics-endpoint.metadata_current-default-0.15.0-dev.0',
path: '/_transform/endpoint.metadata_current-default-0.15.0-dev.0',
ignore: [404],
},
],
[
'transport.request',
{
method: 'DELETE',
path: '/index',
ignore: [404],
},
],
@ -157,7 +192,7 @@ describe('test transform install', () => {
'transport.request',
{
method: 'PUT',
path: '/_transform/metrics-endpoint.metadata-default-0.16.0-dev.0',
path: '/_transform/endpoint.metadata-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
@ -166,7 +201,7 @@ describe('test transform install', () => {
'transport.request',
{
method: 'PUT',
path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
@ -175,14 +210,14 @@ describe('test transform install', () => {
'transport.request',
{
method: 'POST',
path: '/_transform/metrics-endpoint.metadata-default-0.16.0-dev.0/_start',
path: '/_transform/endpoint.metadata-default-0.16.0-dev.0/_start',
},
],
[
'transport.request',
{
method: 'POST',
path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start',
},
],
]);
@ -198,15 +233,15 @@ describe('test transform install', () => {
type: 'ingest_pipeline',
},
{
id: 'metrics-endpoint.metadata_current-default-0.15.0-dev.0',
id: 'endpoint.metadata_current-default-0.15.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
id: 'endpoint.metadata-default-0.16.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
id: 'endpoint.metadata_current-default-0.16.0-dev.0',
type: 'transform',
},
],
@ -222,11 +257,11 @@ describe('test transform install', () => {
type: 'ingest_pipeline',
},
{
id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0',
id: 'endpoint.metadata_current-default-0.16.0-dev.0',
type: 'transform',
},
{
id: 'metrics-endpoint.metadata-default-0.16.0-dev.0',
id: 'endpoint.metadata-default-0.16.0-dev.0',
type: 'transform',
},
],
@ -263,7 +298,7 @@ describe('test transform install', () => {
>)
);
legacyScopedClusterClient.callAsCurrentUser = jest.fn();
await installTransformForDataset(
await installTransform(
({
name: 'endpoint',
version: '0.16.0-dev.0',
@ -284,7 +319,7 @@ describe('test transform install', () => {
},
],
} as unknown) as RegistryPackage,
['endpoint-0.16.0-dev.0/dataset/metadata_current/elasticsearch/transform/default.json'],
['endpoint-0.16.0-dev.0/elasticsearch/transform/metadata_current/default.json'],
legacyScopedClusterClient.callAsCurrentUser,
savedObjectsClient
);
@ -294,7 +329,7 @@ describe('test transform install', () => {
'transport.request',
{
method: 'PUT',
path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0',
path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0',
query: 'defer_validation=true',
body: '{"content": "data"}',
},
@ -303,7 +338,7 @@ describe('test transform install', () => {
'transport.request',
{
method: 'POST',
path: '/_transform/metrics-endpoint.metadata_current-default-0.16.0-dev.0/_start',
path: '/_transform/endpoint.metadata_current-default-0.16.0-dev.0/_start',
},
],
]);
@ -313,7 +348,7 @@ describe('test transform install', () => {
'endpoint',
{
installed_es: [
{ id: 'metrics-endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' },
{ id: 'endpoint.metadata_current-default-0.16.0-dev.0', type: 'transform' },
],
},
],
@ -324,7 +359,7 @@ describe('test transform install', () => {
const previousInstallation: Installation = ({
installed_es: [
{
id: 'metrics-endpoint.metadata-current-default-0.15.0-dev.0',
id: 'endpoint.metadata-current-default-0.15.0-dev.0',
type: ElasticsearchAssetType.transform,
},
],
@ -346,7 +381,26 @@ describe('test transform install', () => {
} as unknown) as SavedObject<Installation>)
);
await installTransformForDataset(
legacyScopedClusterClient.callAsCurrentUser.mockReturnValueOnce(
Promise.resolve({
count: 1,
transforms: [
{
dest: {
index: 'index',
},
},
],
} as {
count: number;
transforms: Array<{
dest: {
index: string;
};
}>;
})
);
await installTransform(
({
name: 'endpoint',
version: '0.16.0-dev.0',
@ -387,11 +441,18 @@ describe('test transform install', () => {
);
expect(legacyScopedClusterClient.callAsCurrentUser.mock.calls).toEqual([
[
'transport.request',
{
method: 'GET',
path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0',
},
],
[
'transport.request',
{
method: 'POST',
path: '/_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0/_stop',
path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0/_stop',
query: 'force=true',
ignore: [404],
},
@ -401,7 +462,15 @@ describe('test transform install', () => {
{
method: 'DELETE',
query: 'force=true',
path: '/_transform/metrics-endpoint.metadata-current-default-0.15.0-dev.0',
path: '/_transform/endpoint.metadata-current-default-0.15.0-dev.0',
ignore: [404],
},
],
[
'transport.request',
{
method: 'DELETE',
path: '/index',
ignore: [404],
},
],

4
x-pack/plugins/ingest_manager/server/services/epm/packages/install.ts
View file

@ -44,7 +44,7 @@ import { updateCurrentWriteIndices } from '../elasticsearch/template/template';
import { deleteKibanaSavedObjectsAssets, removeInstallation } from './remove';
import { IngestManagerError, PackageOutdatedError } from '../../../errors';
import { getPackageSavedObjects } from './get';
import { installTransformForDataset } from '../elasticsearch/transform/install';
import { installTransform } from '../elasticsearch/transform/install';
import { appContextService } from '../../app_context';
export async function installLatestPackage(options: {
@ -325,7 +325,7 @@ export async function installPackage({
// update current backing indices of each data stream
await updateCurrentWriteIndices(callCluster, installedTemplates);
const installedTransforms = await installTransformForDataset(
const installedTransforms = await installTransform(
registryPackageInfo,
paths,
callCluster,

4
x-pack/plugins/security_solution/common/endpoint/constants.ts
View file

@ -7,8 +7,8 @@
export const eventsIndexPattern = 'logs-endpoint.events.*';
export const alertsIndexPattern = 'logs-endpoint.alerts-*';
export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current-*';
export const metadataTransformPrefix = 'metrics-endpoint.metadata-current-default';
export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*';
export const metadataTransformPrefix = 'endpoint.metadata_current-default';
export const policyIndexPattern = 'metrics-endpoint.policy-*';
export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
export const LIMITED_CONCURRENCY_ENDPOINT_ROUTE_TAG = 'endpoint:limited-concurrency';

3
x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts
View file

@ -42,6 +42,7 @@ import {
} from '../../../../../ingest_manager/common/types/models';
import { createV1SearchResponse, createV2SearchResponse } from './support/test_support';
import { PackageService } from '../../../../../ingest_manager/server/services';
import { metadataTransformPrefix } from '../../../../common/endpoint/constants';
describe('test endpoint route', () => {
let routerMock: jest.Mocked<IRouter>;
@ -175,7 +176,7 @@ describe('test endpoint route', () => {
type: ElasticsearchAssetType.indexTemplate,
},
{
id: 'metrics-endpoint.metadata-current-default-0.16.0-dev.0',
id: `${metadataTransformPrefix}-0.16.0-dev.0`,
type: ElasticsearchAssetType.transform,
},
])

4
x-pack/test/ingest_manager_api_integration/apis/epm/install_remove_assets.ts
View file

@ -88,7 +88,7 @@ export default function (providerContext: FtrProviderContext) {
it('should have installed the transform components', async function () {
const res = await es.transport.request({
method: 'GET',
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
path: `/_transform/${pkgName}-test-default-${pkgVersion}`,
});
expect(res.statusCode).equal(200);
});
@ -253,7 +253,7 @@ export default function (providerContext: FtrProviderContext) {
const res = await es.transport.request(
{
method: 'GET',
path: `/_transform/${logsTemplateName}-default-${pkgVersion}`,
path: `/_transform/${pkgName}-test-default-${pkgVersion}`,
},
{
ignore: [404],

0
x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/dataset/test_logs/elasticsearch/transform/default.json → x-pack/test/ingest_manager_api_integration/apis/fixtures/test_packages/all_assets/0.1.0/elasticsearch/transform/test/default.json
View file

14
x-pack/test/security_solution_endpoint/apps/endpoint/endpoint_list.ts
View file

@ -8,7 +8,6 @@ import expect from '@kbn/expect';
import { FtrProviderContext } from '../../ftr_provider_context';
import {
deleteMetadataCurrentStream,
deleteMetadataStream,
deleteAllDocsFromMetadataCurrentIndex,
} from '../../../security_solution_endpoint_api_int/apis/data_stream_helper';
@ -73,13 +72,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
describe('when initially navigating to page', () => {
before(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
await pageObjects.endpoint.navigateToEndpointList();
});
after(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
});
@ -88,8 +85,8 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
});
it('finds data after load and polling', async () => {
await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true });
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 1100);
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
await pageObjects.endpoint.waitForTableToHaveData('endpointListTable', 100000);
const tableData = await pageObjects.endpointPageUtils.tableData('endpointListTable');
expect(tableData).to.eql(expectedData);
});
@ -97,12 +94,12 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
describe('when there is data,', () => {
before(async () => {
await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true });
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
await sleep(100000);
await pageObjects.endpoint.navigateToEndpointList();
});
after(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
});
@ -215,12 +212,11 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => {
describe('displays the correct table data for the kql queries', () => {
before(async () => {
await esArchiver.load('endpoint/metadata/destination_index', { useCreate: true });
await esArchiver.load('endpoint/metadata/api_feature', { useCreate: true });
await pageObjects.endpoint.navigateToEndpointList();
});
after(async () => {
await deleteMetadataStream(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
});
it('for the kql query: na, table shows an empty list', async () => {

4
x-pack/test/security_solution_endpoint_api_int/apis/data_stream_helper.ts
View file

@ -51,10 +51,6 @@ export async function deleteMetadataStream(getService: (serviceName: 'es') => Cl
await deleteDataStream(getService, metadataIndexPattern);
}
export async function deleteMetadataCurrentStream(getService: (serviceName: 'es') => Client) {
await deleteDataStream(getService, metadataCurrentIndexPattern);
}
export async function deleteAllDocsFromMetadataIndex(getService: (serviceName: 'es') => Client) {
await deleteAllDocsFromIndex(getService, metadataIndexPattern);
}

3
x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts
View file

@ -7,7 +7,6 @@ import expect from '@kbn/expect/expect.js';
import { FtrProviderContext } from '../ftr_provider_context';
import {
deleteAllDocsFromMetadataCurrentIndex,
deleteMetadataCurrentStream,
deleteAllDocsFromMetadataIndex,
deleteMetadataStream,
} from './data_stream_helper';
@ -29,7 +28,6 @@ export default function ({ getService }: FtrProviderContext) {
it('metadata api should return empty result when index is empty', async () => {
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
const { body } = await supertest
.post(`${METADATA_REQUEST_ROUTE}`)
@ -54,7 +52,6 @@ export default function ({ getService }: FtrProviderContext) {
after(async () => {
await deleteMetadataStream(getService);
await deleteAllDocsFromMetadataIndex(getService);
await deleteMetadataCurrentStream(getService);
await deleteAllDocsFromMetadataCurrentIndex(getService);
});
it('metadata api should return one entry for each host with default paging', async () => {