From 052dfe9f9a7132a9a13b709e862435065a44420e Mon Sep 17 00:00:00 2001 From: Jen Huang Date: Wed, 17 Jun 2020 12:23:31 -0700 Subject: [PATCH] [Ingest Manager] Replace `datasources` with `inputs` when generating agent config (#69226) * Adjust agent config generation, change `datasources` to `inputs` * Add dataset.type * Remove dead code * Revert "Add dataset.type" This reverts commit fbcf50cbe2b6a93536386c82d754a6a7c7f2b8e9. * Update endpoint policy test assertion --- .../common/services/config_to_yaml.ts | 5 +- .../datasource_to_agent_datasource.test.ts | 190 ------------------ .../datasource_to_agent_datasource.ts | 60 ------ .../datasources_to_agent_inputs.test.ts | 158 +++++++++++++++ .../services/datasources_to_agent_inputs.ts | 64 ++++++ .../ingest_manager/common/services/index.ts | 2 +- .../common/types/models/agent_config.ts | 38 ++-- .../ingest_manager/services/index.ts | 2 +- .../server/services/agent_config.test.ts | 6 +- .../server/services/agent_config.ts | 6 +- .../ingest_manager/server/types/index.tsx | 2 +- .../apps/endpoint/policy_details.ts | 152 ++++++-------- 12 files changed, 307 insertions(+), 378 deletions(-) delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts delete mode 100644 x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts create mode 100644 x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts diff --git a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts index 9dfd76b9ddd2..a3bef72e8db5 100644 --- a/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts +++ b/x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts @@ -11,10 +11,11 @@ const CONFIG_KEYS_ORDER = [ 'name', 'revision', 'type', - 'outputs', 'settings', - 'datasources', + 'outputs', + 'inputs', 'enabled', + 'use_output', 'package', 'input', ]; diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts deleted file mode 100644 index d319ba2beddf..000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, DatasourceInput } from '../types'; -import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; - -describe('Ingest Manager - storedDatasourceToAgentDatasource', () => { - const mockDatasource: Datasource = { - id: 'some-uuid', - name: 'mock-datasource', - description: '', - created_at: '', - created_by: '', - updated_at: '', - updated_by: '', - config_id: '', - enabled: true, - output_id: '', - namespace: 'default', - inputs: [], - revision: 1, - }; - - const mockInput: DatasourceInput = { - type: 'test-logs', - enabled: true, - vars: { - inputVar: { value: 'input-value' }, - inputVar2: { value: undefined }, - inputVar3: { - type: 'yaml', - value: 'testField: test', - }, - inputVar4: { value: '' }, - }, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - vars: { - fooVar: { value: 'foo-value' }, - fooVar2: { value: [1, 2] }, - }, - agent_stream: { - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - vars: { - barVar: { value: 'bar-value' }, - barVar2: { value: [1, 2] }, - barVar3: { - type: 'yaml', - value: - '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', - }, - barVar4: { - type: 'yaml', - value: '', - }, - barVar5: { - type: 'yaml', - value: 'testField: test\n invalidSpacing: foo', - }, - }, - }, - ], - }; - - it('returns agent datasource config for datasource with no inputs', () => { - expect(storedDatasourceToAgentDatasource(mockDatasource)).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - package: { - name: 'mock-package', - title: 'Mock package', - version: '0.0.0', - }, - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - package: { - name: 'mock-package', - version: '0.0.0', - }, - inputs: [], - }); - }); - - it('returns agent datasource config with flattened input and package stream', () => { - expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - { - id: 'test-logs-bar', - enabled: true, - dataset: 'bar', - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled streams', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [ - { - ...mockInput, - streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], - }, - ], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [ - { - type: 'test-logs', - enabled: true, - streams: [ - { - id: 'test-logs-foo', - enabled: true, - dataset: 'foo', - fooKey: 'fooValue1', - fooKey2: ['fooValue2'], - }, - ], - }, - ], - }); - }); - - it('returns agent datasource config without disabled inputs', () => { - expect( - storedDatasourceToAgentDatasource({ - ...mockDatasource, - inputs: [{ ...mockInput, enabled: false }], - }) - ).toEqual({ - id: 'some-uuid', - name: 'mock-datasource', - namespace: 'default', - enabled: true, - use_output: 'default', - inputs: [], - }); - }); -}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts deleted file mode 100644 index 2a8b687675bf..000000000000 --- a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Datasource, FullAgentConfigDatasource } from '../types'; -import { DEFAULT_OUTPUT } from '../constants'; - -export const storedDatasourceToAgentDatasource = ( - datasource: Datasource -): FullAgentConfigDatasource => { - const { id, name, namespace, enabled, package: pkg, inputs } = datasource; - - const fullDatasource: FullAgentConfigDatasource = { - id: id || name, - name, - namespace, - enabled, - use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now - inputs: inputs - .filter((input) => input.enabled) - .map((input) => { - const fullInput = { - ...input, - ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - streams: input.streams - .filter((stream) => stream.enabled) - .map((stream) => { - const fullStream = { - ...stream, - ...stream.agent_stream, - ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { - acc[key] = value; - return acc; - }, {} as { [k: string]: any }), - }; - delete fullStream.agent_stream; - delete fullStream.vars; - delete fullStream.config; - return fullStream; - }), - }; - delete fullInput.vars; - delete fullInput.config; - return fullInput; - }), - }; - - if (pkg) { - fullDatasource.package = { - name: pkg.name, - version: pkg.version, - }; - } - - return fullDatasource; -}; diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts new file mode 100644 index 000000000000..df94168ec88d --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts @@ -0,0 +1,158 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, DatasourceInput } from '../types'; +import { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; + +describe('Ingest Manager - storedDatasourcesToAgentInputs', () => { + const mockDatasource: Datasource = { + id: 'some-uuid', + name: 'mock-datasource', + description: '', + created_at: '', + created_by: '', + updated_at: '', + updated_by: '', + config_id: '', + enabled: true, + output_id: '', + namespace: 'default', + inputs: [], + revision: 1, + }; + + const mockInput: DatasourceInput = { + type: 'test-logs', + enabled: true, + vars: { + inputVar: { value: 'input-value' }, + inputVar2: { value: undefined }, + inputVar3: { + type: 'yaml', + value: 'testField: test', + }, + inputVar4: { value: '' }, + }, + streams: [ + { + id: 'test-logs-foo', + enabled: true, + dataset: 'foo', + vars: { + fooVar: { value: 'foo-value' }, + fooVar2: { value: [1, 2] }, + }, + agent_stream: { + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + }, + { + id: 'test-logs-bar', + enabled: true, + dataset: 'bar', + vars: { + barVar: { value: 'bar-value' }, + barVar2: { value: [1, 2] }, + barVar3: { + type: 'yaml', + value: + '- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2', + }, + barVar4: { + type: 'yaml', + value: '', + }, + barVar5: { + type: 'yaml', + value: 'testField: test\n invalidSpacing: foo', + }, + }, + }, + ], + }; + + it('returns no inputs for datasource with no inputs, or only disabled inputs', () => { + expect(storedDatasourcesToAgentInputs([mockDatasource])).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + package: { + name: 'mock-package', + title: 'Mock package', + version: '0.0.0', + }, + }, + ]) + ).toEqual([]); + + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [{ ...mockInput, enabled: false }], + }, + ]) + ).toEqual([]); + }); + + it('returns agent inputs', () => { + expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + { + id: 'test-logs-bar', + dataset: { name: 'bar' }, + }, + ], + }, + ]); + }); + + it('returns agent inputs without disabled streams', () => { + expect( + storedDatasourcesToAgentInputs([ + { + ...mockDatasource, + inputs: [ + { + ...mockInput, + streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }], + }, + ], + }, + ]) + ).toEqual([ + { + id: 'some-uuid', + name: 'mock-datasource', + type: 'test-logs', + dataset: { namespace: 'default' }, + use_output: 'default', + streams: [ + { + id: 'test-logs-foo', + dataset: { name: 'foo' }, + fooKey: 'fooValue1', + fooKey2: ['fooValue2'], + }, + ], + }, + ]); + }); +}); diff --git a/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts new file mode 100644 index 000000000000..d5a752e817b4 --- /dev/null +++ b/x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Datasource, FullAgentConfigInput, FullAgentConfigInputStream } from '../types'; +import { DEFAULT_OUTPUT } from '../constants'; + +export const storedDatasourcesToAgentInputs = ( + datasources: Datasource[] +): FullAgentConfigInput[] => { + const fullInputs: FullAgentConfigInput[] = []; + + datasources.forEach((datasource) => { + if (!datasource.enabled || !datasource.inputs || !datasource.inputs.length) { + return; + } + datasource.inputs.forEach((input) => { + if (!input.enabled) { + return; + } + + const fullInput: FullAgentConfigInput = { + id: datasource.id || datasource.name, + name: datasource.name, + type: input.type, + dataset: { namespace: datasource.namespace || 'default' }, + use_output: DEFAULT_OUTPUT.name, + ...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + streams: input.streams + .filter((stream) => stream.enabled) + .map((stream) => { + const fullStream: FullAgentConfigInputStream = { + id: stream.id, + dataset: { name: stream.dataset }, + ...stream.agent_stream, + ...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => { + acc[key] = value; + return acc; + }, {} as { [k: string]: any }), + }; + if (stream.processors) { + fullStream.processors = stream.processors; + } + return fullStream; + }), + }; + + if (datasource.package) { + fullInput.package = { + name: datasource.package.name, + version: datasource.package.version, + }; + } + + fullInputs.push(fullInput); + }); + }); + + return fullInputs; +}; diff --git a/x-pack/plugins/ingest_manager/common/services/index.ts b/x-pack/plugins/ingest_manager/common/services/index.ts index c595c9a52f66..e53d97972fa2 100644 --- a/x-pack/plugins/ingest_manager/common/services/index.ts +++ b/x-pack/plugins/ingest_manager/common/services/index.ts @@ -7,7 +7,7 @@ import * as AgentStatusKueryHelper from './agent_status'; export * from './routes'; export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config'; -export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource'; +export { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs'; export { configToYaml } from './config_to_yaml'; export { AgentStatusKueryHelper }; export { decodeCloudId } from './decode_cloud_id'; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts index 7547f56237ee..36b3176ffa41 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_config.ts @@ -3,12 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { - Datasource, - DatasourcePackage, - DatasourceInput, - DatasourceInputStream, -} from './datasource'; +import { Datasource, DatasourcePackage, DatasourceInputStream } from './datasource'; import { Output } from './output'; export enum AgentConfigStatus { @@ -35,23 +30,22 @@ export interface AgentConfig extends NewAgentConfig { export type AgentConfigSOAttributes = Omit; -export type FullAgentConfigDatasource = Pick< - Datasource, - 'id' | 'name' | 'namespace' | 'enabled' -> & { - package?: Pick; - use_output: string; - inputs: Array< - Omit & { - streams: Array< - Omit & { - [key: string]: any; - } - >; - } - >; +export type FullAgentConfigInputStream = Pick & { + dataset: { name: string }; + [key: string]: any; }; +export interface FullAgentConfigInput { + id: string; + name: string; + type: string; + dataset: { namespace: string }; + use_output: string; + package?: Pick; + streams: FullAgentConfigInputStream[]; + [key: string]: any; +} + export interface FullAgentConfig { id: string; outputs: { @@ -59,7 +53,7 @@ export interface FullAgentConfig { [key: string]: any; }; }; - datasources: FullAgentConfigDatasource[]; + inputs: FullAgentConfigInput[]; revision?: number; settings?: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts index 6dbc8d67caae..ece7aef2c247 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts @@ -19,7 +19,7 @@ export { settingsRoutesService, appRoutesService, packageToConfigDatasourceInputs, - storedDatasourceToAgentDatasource, + storedDatasourcesToAgentInputs, configToYaml, AgentStatusKueryHelper, } from '../../../../common'; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts index 17758f6e3d7f..c46e648ad088 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.test.ts @@ -59,7 +59,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -88,7 +88,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { @@ -118,7 +118,7 @@ describe('agent config', () => { api_key: undefined, }, }, - datasources: [], + inputs: [], revision: 1, settings: { monitoring: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_config.ts b/x-pack/plugins/ingest_manager/server/services/agent_config.ts index 18d5d8dedfb1..9e0386de7476 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_config.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_config.ts @@ -20,7 +20,7 @@ import { AgentConfigStatus, ListWithKuery, } from '../types'; -import { DeleteAgentConfigResponse, storedDatasourceToAgentDatasource } from '../../common'; +import { DeleteAgentConfigResponse, storedDatasourcesToAgentInputs } from '../../common'; import { listAgents } from './agents'; import { datasourceService } from './datasource'; import { outputService } from './output'; @@ -375,9 +375,7 @@ class AgentConfigService { {} as FullAgentConfig['outputs'] ), }, - datasources: (config.datasources as Datasource[]) - .filter((datasource) => datasource.enabled) - .map((ds) => storedDatasourceToAgentDatasource(ds)), + inputs: storedDatasourcesToAgentInputs(config.datasources as Datasource[]), revision: config.revision, ...(config.monitoring_enabled && config.monitoring_enabled.length > 0 ? { diff --git a/x-pack/plugins/ingest_manager/server/types/index.tsx b/x-pack/plugins/ingest_manager/server/types/index.tsx index 2218d967fa8a..2b543490ca8d 100644 --- a/x-pack/plugins/ingest_manager/server/types/index.tsx +++ b/x-pack/plugins/ingest_manager/server/types/index.tsx @@ -20,7 +20,7 @@ export { Datasource, NewDatasource, DatasourceSOAttributes, - FullAgentConfigDatasource, + FullAgentConfigInput, FullAgentConfig, AgentConfig, AgentConfigSOAttributes, diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index 25fb477b5a99..036f82a591fb 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -99,107 +99,71 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { ); expect(agentFullConfig).to.eql({ - datasources: [ + inputs: [ { - enabled: true, id: policyInfo.datasource.id, - inputs: [ - { - enabled: true, - policy: { - linux: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - file: false, - network: true, - process: true, - }, - logging: { - file: 'info', - stdout: 'debug', - }, - }, - mac: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - file: false, - network: true, - process: true, - }, - logging: { - file: 'info', - stdout: 'debug', - }, - malware: { - mode: 'detect', - }, - }, - windows: { - advanced: { - elasticsearch: { - indices: { - control: 'control-index', - event: 'event-index', - logging: 'logging-index', - }, - kernel: { - connect: true, - process: true, - }, - }, - }, - events: { - dll_and_driver_load: true, - dns: true, - file: false, - network: true, - process: true, - registry: true, - security: true, - }, - logging: { - file: 'info', - stdout: 'debug', - }, - malware: { - mode: 'prevent', - }, - }, - }, - streams: [], - type: 'endpoint', - }, - ], + dataset: { namespace: 'default' }, name: 'Protect East Coast', - namespace: 'default', package: { name: 'endpoint', version: policyInfo.packageInfo.version, }, + policy: { + linux: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', + }, + kernel: { connect: true, process: true }, + }, + }, + events: { file: false, network: true, process: true }, + logging: { file: 'info', stdout: 'debug' }, + }, + mac: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', + }, + kernel: { connect: true, process: true }, + }, + }, + events: { file: false, network: true, process: true }, + logging: { file: 'info', stdout: 'debug' }, + malware: { mode: 'detect' }, + }, + windows: { + advanced: { + elasticsearch: { + indices: { + control: 'control-index', + event: 'event-index', + logging: 'logging-index', + }, + kernel: { connect: true, process: true }, + }, + }, + events: { + dll_and_driver_load: true, + dns: true, + file: false, + network: true, + process: true, + registry: true, + security: true, + }, + logging: { file: 'info', stdout: 'debug' }, + malware: { mode: 'prevent' }, + }, + }, + streams: [], + type: 'endpoint', use_output: 'default', }, ],