maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Ingest Manager] Replace datasources with inputs when generating agent config (#69226)

Browse source 
* Adjust agent config generation, change `datasources` to `inputs`

* Add dataset.type

* Remove dead code

* Revert "Add dataset.type"

This reverts commit fbcf50cbe2.

* Update endpoint policy test assertion
This commit is contained in:
Jen Huang 2020-06-17 12:23:31 -07:00 committed by GitHub
parent bb03b75416
commit 052dfe9f9a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 307 additions and 378 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
x-pack/plugins/ingest_manager/common/services/config_to_yaml.ts
View file

@ -11,10 +11,11 @@ const CONFIG_KEYS_ORDER = [
'name',
'revision',
'type',
'outputs',
'settings',
'datasources',
'outputs',
'inputs',
'enabled',
'use_output',
'package',
'input',
];

190
x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts
View file

@ -1,190 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Datasource, DatasourceInput } from '../types';
import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
const mockDatasource: Datasource = {
id: 'some-uuid',
name: 'mock-datasource',
description: '',
created_at: '',
created_by: '',
updated_at: '',
updated_by: '',
config_id: '',
enabled: true,
output_id: '',
namespace: 'default',
inputs: [],
revision: 1,
};
const mockInput: DatasourceInput = {
type: 'test-logs',
enabled: true,
vars: {
inputVar: { value: 'input-value' },
inputVar2: { value: undefined },
inputVar3: {
type: 'yaml',
value: 'testField: test',
},
inputVar4: { value: '' },
},
streams: [
{
id: 'test-logs-foo',
enabled: true,
dataset: 'foo',
vars: {
fooVar: { value: 'foo-value' },
fooVar2: { value: [1, 2] },
},
agent_stream: {
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
},
{
id: 'test-logs-bar',
enabled: true,
dataset: 'bar',
vars: {
barVar: { value: 'bar-value' },
barVar2: { value: [1, 2] },
barVar3: {
type: 'yaml',
value:
'- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2',
},
barVar4: {
type: 'yaml',
value: '',
},
barVar5: {
type: 'yaml',
value: 'testField: test\n invalidSpacing: foo',
},
},
},
],
};
it('returns agent datasource config for datasource with no inputs', () => {
expect(storedDatasourceToAgentDatasource(mockDatasource)).toEqual({
id: 'some-uuid',
name: 'mock-datasource',
namespace: 'default',
enabled: true,
use_output: 'default',
inputs: [],
});
expect(
storedDatasourceToAgentDatasource({
...mockDatasource,
package: {
name: 'mock-package',
title: 'Mock package',
version: '0.0.0',
},
})
).toEqual({
id: 'some-uuid',
name: 'mock-datasource',
namespace: 'default',
enabled: true,
use_output: 'default',
package: {
name: 'mock-package',
version: '0.0.0',
},
inputs: [],
});
});
it('returns agent datasource config with flattened input and package stream', () => {
expect(storedDatasourceToAgentDatasource({ ...mockDatasource, inputs: [mockInput] })).toEqual({
id: 'some-uuid',
name: 'mock-datasource',
namespace: 'default',
enabled: true,
use_output: 'default',
inputs: [
{
type: 'test-logs',
enabled: true,
streams: [
{
id: 'test-logs-foo',
enabled: true,
dataset: 'foo',
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
{
id: 'test-logs-bar',
enabled: true,
dataset: 'bar',
},
],
},
],
});
});
it('returns agent datasource config without disabled streams', () => {
expect(
storedDatasourceToAgentDatasource({
...mockDatasource,
inputs: [
{
...mockInput,
streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }],
},
],
})
).toEqual({
id: 'some-uuid',
name: 'mock-datasource',
namespace: 'default',
enabled: true,
use_output: 'default',
inputs: [
{
type: 'test-logs',
enabled: true,
streams: [
{
id: 'test-logs-foo',
enabled: true,
dataset: 'foo',
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
],
},
],
});
});
it('returns agent datasource config without disabled inputs', () => {
expect(
storedDatasourceToAgentDatasource({
...mockDatasource,
inputs: [{ ...mockInput, enabled: false }],
})
).toEqual({
id: 'some-uuid',
name: 'mock-datasource',
namespace: 'default',
enabled: true,
use_output: 'default',
inputs: [],
});
});
});

60
x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts
View file

@ -1,60 +0,0 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Datasource, FullAgentConfigDatasource } from '../types';
import { DEFAULT_OUTPUT } from '../constants';
export const storedDatasourceToAgentDatasource = (
datasource: Datasource
): FullAgentConfigDatasource => {
const { id, name, namespace, enabled, package: pkg, inputs } = datasource;
const fullDatasource: FullAgentConfigDatasource = {
id: id || name,
name,
namespace,
enabled,
use_output: DEFAULT_OUTPUT.name, // TODO: hardcoded to default output for now
inputs: inputs
.filter((input) => input.enabled)
.map((input) => {
const fullInput = {
...input,
...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
return acc;
}, {} as { [k: string]: any }),
streams: input.streams
.filter((stream) => stream.enabled)
.map((stream) => {
const fullStream = {
...stream,
...stream.agent_stream,
...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
return acc;
}, {} as { [k: string]: any }),
};
delete fullStream.agent_stream;
delete fullStream.vars;
delete fullStream.config;
return fullStream;
}),
};
delete fullInput.vars;
delete fullInput.config;
return fullInput;
}),
};
if (pkg) {
fullDatasource.package = {
name: pkg.name,
version: pkg.version,
};
}
return fullDatasource;
};

158
x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.test.ts Normal file
View file

@ -0,0 +1,158 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Datasource, DatasourceInput } from '../types';
import { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs';
describe('Ingest Manager - storedDatasourcesToAgentInputs', () => {
const mockDatasource: Datasource = {
id: 'some-uuid',
name: 'mock-datasource',
description: '',
created_at: '',
created_by: '',
updated_at: '',
updated_by: '',
config_id: '',
enabled: true,
output_id: '',
namespace: 'default',
inputs: [],
revision: 1,
};
const mockInput: DatasourceInput = {
type: 'test-logs',
enabled: true,
vars: {
inputVar: { value: 'input-value' },
inputVar2: { value: undefined },
inputVar3: {
type: 'yaml',
value: 'testField: test',
},
inputVar4: { value: '' },
},
streams: [
{
id: 'test-logs-foo',
enabled: true,
dataset: 'foo',
vars: {
fooVar: { value: 'foo-value' },
fooVar2: { value: [1, 2] },
},
agent_stream: {
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
},
{
id: 'test-logs-bar',
enabled: true,
dataset: 'bar',
vars: {
barVar: { value: 'bar-value' },
barVar2: { value: [1, 2] },
barVar3: {
type: 'yaml',
value:
'- namespace: mockNamespace\n #disabledProp: ["test"]\n anotherProp: test\n- namespace: mockNamespace2\n #disabledProp: ["test2"]\n anotherProp: test2',
},
barVar4: {
type: 'yaml',
value: '',
},
barVar5: {
type: 'yaml',
value: 'testField: test\n invalidSpacing: foo',
},
},
},
],
};
it('returns no inputs for datasource with no inputs, or only disabled inputs', () => {
expect(storedDatasourcesToAgentInputs([mockDatasource])).toEqual([]);
expect(
storedDatasourcesToAgentInputs([
{
...mockDatasource,
package: {
name: 'mock-package',
title: 'Mock package',
version: '0.0.0',
},
},
])
).toEqual([]);
expect(
storedDatasourcesToAgentInputs([
{
...mockDatasource,
inputs: [{ ...mockInput, enabled: false }],
},
])
).toEqual([]);
});
it('returns agent inputs', () => {
expect(storedDatasourcesToAgentInputs([{ ...mockDatasource, inputs: [mockInput] }])).toEqual([
{
id: 'some-uuid',
name: 'mock-datasource',
type: 'test-logs',
dataset: { namespace: 'default' },
use_output: 'default',
streams: [
{
id: 'test-logs-foo',
dataset: { name: 'foo' },
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
{
id: 'test-logs-bar',
dataset: { name: 'bar' },
},
],
},
]);
});
it('returns agent inputs without disabled streams', () => {
expect(
storedDatasourcesToAgentInputs([
{
...mockDatasource,
inputs: [
{
...mockInput,
streams: [{ ...mockInput.streams[0] }, { ...mockInput.streams[1], enabled: false }],
},
],
},
])
).toEqual([
{
id: 'some-uuid',
name: 'mock-datasource',
type: 'test-logs',
dataset: { namespace: 'default' },
use_output: 'default',
streams: [
{
id: 'test-logs-foo',
dataset: { name: 'foo' },
fooKey: 'fooValue1',
fooKey2: ['fooValue2'],
},
],
},
]);
});
});

64
x-pack/plugins/ingest_manager/common/services/datasources_to_agent_inputs.ts Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { Datasource, FullAgentConfigInput, FullAgentConfigInputStream } from '../types';
import { DEFAULT_OUTPUT } from '../constants';
export const storedDatasourcesToAgentInputs = (
datasources: Datasource[]
): FullAgentConfigInput[] => {
const fullInputs: FullAgentConfigInput[] = [];
datasources.forEach((datasource) => {
if (!datasource.enabled || !datasource.inputs || !datasource.inputs.length) {
return;
}
datasource.inputs.forEach((input) => {
if (!input.enabled) {
return;
}
const fullInput: FullAgentConfigInput = {
id: datasource.id || datasource.name,
name: datasource.name,
type: input.type,
dataset: { namespace: datasource.namespace || 'default' },
use_output: DEFAULT_OUTPUT.name,
...Object.entries(input.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
return acc;
}, {} as { [k: string]: any }),
streams: input.streams
.filter((stream) => stream.enabled)
.map((stream) => {
const fullStream: FullAgentConfigInputStream = {
id: stream.id,
dataset: { name: stream.dataset },
...stream.agent_stream,
...Object.entries(stream.config || {}).reduce((acc, [key, { value }]) => {
acc[key] = value;
return acc;
}, {} as { [k: string]: any }),
};
if (stream.processors) {
fullStream.processors = stream.processors;
}
return fullStream;
}),
};
if (datasource.package) {
fullInput.package = {
name: datasource.package.name,
version: datasource.package.version,
};
}
fullInputs.push(fullInput);
});
});
return fullInputs;
};

2
x-pack/plugins/ingest_manager/common/services/index.ts
View file

@ -7,7 +7,7 @@ import * as AgentStatusKueryHelper from './agent_status';
export * from './routes';
export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config';
export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
export { storedDatasourcesToAgentInputs } from './datasources_to_agent_inputs';
export { configToYaml } from './config_to_yaml';
export { AgentStatusKueryHelper };
export { decodeCloudId } from './decode_cloud_id';

38
x-pack/plugins/ingest_manager/common/types/models/agent_config.ts
View file

@ -3,12 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import {
Datasource,
DatasourcePackage,
DatasourceInput,
DatasourceInputStream,
} from './datasource';
import { Datasource, DatasourcePackage, DatasourceInputStream } from './datasource';
import { Output } from './output';
export enum AgentConfigStatus {
@ -35,23 +30,22 @@ export interface AgentConfig extends NewAgentConfig {
export type AgentConfigSOAttributes = Omit<AgentConfig, 'id'>;
export type FullAgentConfigDatasource = Pick<
Datasource,
'id' | 'name' | 'namespace' | 'enabled'
> & {
package?: Pick<DatasourcePackage, 'name' | 'version'>;
use_output: string;
inputs: Array<
Omit<DatasourceInput, 'streams'> & {
streams: Array<
Omit<DatasourceInputStream, 'config'> & {
[key: string]: any;
}
>;
}
>;
export type FullAgentConfigInputStream = Pick<DatasourceInputStream, 'id' | 'processors'> & {
dataset: { name: string };
[key: string]: any;
};
export interface FullAgentConfigInput {
id: string;
name: string;
type: string;
dataset: { namespace: string };
use_output: string;
package?: Pick<DatasourcePackage, 'name' | 'version'>;
streams: FullAgentConfigInputStream[];
[key: string]: any;
}
export interface FullAgentConfig {
id: string;
outputs: {
@ -59,7 +53,7 @@ export interface FullAgentConfig {
[key: string]: any;
};
};
datasources: FullAgentConfigDatasource[];
inputs: FullAgentConfigInput[];
revision?: number;
settings?: {
monitoring: {

2
x-pack/plugins/ingest_manager/public/applications/ingest_manager/services/index.ts
View file

@ -19,7 +19,7 @@ export {
settingsRoutesService,
appRoutesService,
packageToConfigDatasourceInputs,
storedDatasourceToAgentDatasource,
storedDatasourcesToAgentInputs,
configToYaml,
AgentStatusKueryHelper,
} from '../../../../common';

6
x-pack/plugins/ingest_manager/server/services/agent_config.test.ts
View file

@ -59,7 +59,7 @@ describe('agent config', () => {
api_key: undefined,
},
},
datasources: [],
inputs: [],
revision: 1,
settings: {
monitoring: {
@ -88,7 +88,7 @@ describe('agent config', () => {
api_key: undefined,
},
},
datasources: [],
inputs: [],
revision: 1,
settings: {
monitoring: {
@ -118,7 +118,7 @@ describe('agent config', () => {
api_key: undefined,
},
},
datasources: [],
inputs: [],
revision: 1,
settings: {
monitoring: {

6
x-pack/plugins/ingest_manager/server/services/agent_config.ts
View file

@ -20,7 +20,7 @@ import {
AgentConfigStatus,
ListWithKuery,
} from '../types';
import { DeleteAgentConfigResponse, storedDatasourceToAgentDatasource } from '../../common';
import { DeleteAgentConfigResponse, storedDatasourcesToAgentInputs } from '../../common';
import { listAgents } from './agents';
import { datasourceService } from './datasource';
import { outputService } from './output';
@ -375,9 +375,7 @@ class AgentConfigService {
{} as FullAgentConfig['outputs']
),
},
datasources: (config.datasources as Datasource[])
.filter((datasource) => datasource.enabled)
.map((ds) => storedDatasourceToAgentDatasource(ds)),
inputs: storedDatasourcesToAgentInputs(config.datasources as Datasource[]),
revision: config.revision,
...(config.monitoring_enabled && config.monitoring_enabled.length > 0
? {

2
x-pack/plugins/ingest_manager/server/types/index.tsx
View file

@ -20,7 +20,7 @@ export {
Datasource,
NewDatasource,
DatasourceSOAttributes,
FullAgentConfigDatasource,
FullAgentConfigInput,
FullAgentConfig,
AgentConfig,
AgentConfigSOAttributes,

152
x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts
View file

@ -99,107 +99,71 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) {
);
expect(agentFullConfig).to.eql({
datasources: [
inputs: [
{
enabled: true,
id: policyInfo.datasource.id,
inputs: [
{
enabled: true,
policy: {
linux: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: {
connect: true,
process: true,
},
},
},
events: {
file: false,
network: true,
process: true,
},
logging: {
file: 'info',
stdout: 'debug',
},
},
mac: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: {
connect: true,
process: true,
},
},
},
events: {
file: false,
network: true,
process: true,
},
logging: {
file: 'info',
stdout: 'debug',
},
malware: {
mode: 'detect',
},
},
windows: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: {
connect: true,
process: true,
},
},
},
events: {
dll_and_driver_load: true,
dns: true,
file: false,
network: true,
process: true,
registry: true,
security: true,
},
logging: {
file: 'info',
stdout: 'debug',
},
malware: {
mode: 'prevent',
},
},
},
streams: [],
type: 'endpoint',
},
],
dataset: { namespace: 'default' },
name: 'Protect East Coast',
namespace: 'default',
package: {
name: 'endpoint',
version: policyInfo.packageInfo.version,
},
policy: {
linux: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: { connect: true, process: true },
},
},
events: { file: false, network: true, process: true },
logging: { file: 'info', stdout: 'debug' },
},
mac: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: { connect: true, process: true },
},
},
events: { file: false, network: true, process: true },
logging: { file: 'info', stdout: 'debug' },
malware: { mode: 'detect' },
},
windows: {
advanced: {
elasticsearch: {
indices: {
control: 'control-index',
event: 'event-index',
logging: 'logging-index',
},
kernel: { connect: true, process: true },
},
},
events: {
dll_and_driver_load: true,
dns: true,
file: false,
network: true,
process: true,
registry: true,
security: true,
},
logging: { file: 'info', stdout: 'debug' },
malware: { mode: 'prevent' },
},
},
streams: [],
type: 'endpoint',
use_output: 'default',
},
],