maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Fleet] Allow to preconfigure alternative ES outputs (on the same cluster) (#111002) (#112711)

Browse source 
Co-authored-by: Nicolas Chaulet <nicolas.chaulet@elastic.co>
This commit is contained in:
Kibana Machine 2021-09-21 16:18:16 -04:00 committed by GitHub
parent 1334789c5d
commit 69b57c0b28
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 1517 additions and 373 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
docs/settings/fleet-settings.asciidoc
View file

@ -86,6 +86,8 @@ Optional properties are:
be changed by updating the {kib} config.
`is_default`:: If `true`, this policy is the default agent policy.
`is_default_fleet_server`:: If `true`, this policy is the default {fleet-server} agent policy.
`data_output_id`:: ID of the output to send data (Need to be identical to `monitoring_output_id`)
`monitoring_output_id`:: ID of the output to send monitoring data. (Need to be identical to `data_output_id`)
`package_policies`:: List of integration policies to add to this policy.
`name`::: (required) Name of the integration policy.
`package`::: (required) Integration that this policy configures
@ -96,6 +98,20 @@ Optional properties are:
integration. Follows the same schema as integration inputs, with the
exception that any object in `vars` can be passed `frozen: true` in order to
prevent that specific `var` from being edited by the user.
| `xpack.fleet.outputs`
| List of ouputs that are configured when the {fleet} app starts.
Required properties are:
`id`:: Unique ID for this output. The ID should be a string.
`name`:: Output name.
`type`:: Type of Output. Currently we only support "elasticsearch".
`hosts`:: Array that contains the list of host for that output.
`config`:: Extra config for that output.
Optional properties are:
`is_default`:: If `true`, this output is the default output.
|===
Example configuration:

4
x-pack/plugins/fleet/common/constants/output.ts
View file

@ -13,8 +13,10 @@ export const outputType = {
Elasticsearch: 'elasticsearch',
} as const;
export const DEFAULT_OUTPUT_ID = 'default';
export const DEFAULT_OUTPUT: NewOutput = {
name: 'default',
name: DEFAULT_OUTPUT_ID,
is_default: true,
type: outputType.Elasticsearch,
hosts: [''],

5
x-pack/plugins/fleet/common/services/package_policies_to_agent_inputs.ts
View file

@ -11,7 +11,8 @@ import type { PackagePolicy, FullAgentPolicyInput, FullAgentPolicyInputStream }
import { DEFAULT_OUTPUT } from '../constants';
export const storedPackagePoliciesToAgentInputs = (
packagePolicies: PackagePolicy[]
packagePolicies: PackagePolicy[],
outputId: string = DEFAULT_OUTPUT.name
): FullAgentPolicyInput[] => {
const fullInputs: FullAgentPolicyInput[] = [];
@ -32,7 +33,7 @@ export const storedPackagePoliciesToAgentInputs = (
data_stream: {
namespace: packagePolicy.namespace || 'default',
},
use_output: DEFAULT_OUTPUT.name,
use_output: outputId,
...(input.compiled_input || {}),
...(input.streams.length
? {

7
x-pack/plugins/fleet/common/types/index.ts
View file

@ -8,7 +8,11 @@
export * from './models';
export * from './rest_spec';
import type { PreconfiguredAgentPolicy, PreconfiguredPackage } from './models/preconfiguration';
import type {
PreconfiguredAgentPolicy,
PreconfiguredPackage,
PreconfiguredOutput,
} from './models/preconfiguration';
export interface FleetConfigType {
enabled: boolean;
@ -26,6 +30,7 @@ export interface FleetConfigType {
};
agentPolicies?: PreconfiguredAgentPolicy[];
packages?: PreconfiguredPackage[];
outputs?: PreconfiguredOutput[];
agentIdVerificationEnabled?: boolean;
}

10
x-pack/plugins/fleet/common/types/models/agent_policy.ts
View file

@ -23,6 +23,8 @@ export interface NewAgentPolicy {
monitoring_enabled?: MonitoringType;
unenroll_timeout?: number;
is_preconfigured?: boolean;
data_output_id?: string;
monitoring_output_id?: string;
}
export interface AgentPolicy extends NewAgentPolicy {
@ -71,12 +73,14 @@ export interface FullAgentPolicyOutputPermissions {
};
}
export type FullAgentPolicyOutput = Pick<Output, 'type' | 'hosts' | 'ca_sha256' | 'api_key'> & {
[key: string]: any;
};
export interface FullAgentPolicy {
id: string;
outputs: {
[key: string]: Pick<Output, 'type' | 'hosts' | 'ca_sha256' | 'api_key'> & {
[key: string]: any;
};
[key: string]: FullAgentPolicyOutput;
};
output_permissions?: {
[output: string]: FullAgentPolicyOutputPermissions;

6
x-pack/plugins/fleet/common/types/models/output.ts
View file

@ -17,11 +17,13 @@ export interface NewOutput {
hosts?: string[];
ca_sha256?: string;
api_key?: string;
config?: Record<string, any>;
config_yaml?: string;
is_preconfigured?: boolean;
}
export type OutputSOAttributes = NewOutput;
export type OutputSOAttributes = NewOutput & {
output_id?: string;
};
export type Output = NewOutput & {
id: string;

5
x-pack/plugins/fleet/common/types/models/preconfiguration.ts
View file

@ -11,6 +11,7 @@ import type {
NewPackagePolicyInput,
} from './package_policy';
import type { NewAgentPolicy } from './agent_policy';
import type { Output } from './output';
export type InputsOverride = Partial<NewPackagePolicyInput> & {
vars?: Array<NewPackagePolicyInput['vars'] & { name: string }>;
@ -29,3 +30,7 @@ export interface PreconfiguredAgentPolicy extends Omit<NewAgentPolicy, 'namespac
}
export type PreconfiguredPackage = Omit<PackagePolicyPackage, 'title'>;
export interface PreconfiguredOutput extends Omit<Output, 'config_yaml'> {
config?: Record<string, unknown>;
}

4
x-pack/plugins/fleet/server/errors/utils.ts
View file

@ -11,6 +11,6 @@ export function isESClientError(error: unknown): error is ResponseError {
return error instanceof ResponseError;
}
export const isElasticsearchVersionConflictError = (error: Error): boolean => {
export function isElasticsearchVersionConflictError(error: Error): boolean {
return isESClientError(error) && error.meta.statusCode === 409;
};
}

7
x-pack/plugins/fleet/server/index.ts
View file

@ -9,7 +9,11 @@ import { schema } from '@kbn/config-schema';
import type { TypeOf } from '@kbn/config-schema';
import type { PluginConfigDescriptor, PluginInitializerContext } from 'src/core/server';
import { PreconfiguredPackagesSchema, PreconfiguredAgentPoliciesSchema } from './types';
import {
PreconfiguredPackagesSchema,
PreconfiguredAgentPoliciesSchema,
PreconfiguredOutputsSchema,
} from './types';
import { FleetPlugin } from './plugin';
@ -113,6 +117,7 @@ export const config: PluginConfigDescriptor = {
}),
packages: PreconfiguredPackagesSchema,
agentPolicies: PreconfiguredAgentPoliciesSchema,
outputs: PreconfiguredOutputsSchema,
agentIdVerificationEnabled: schema.boolean({ defaultValue: true }),
}),
};

4
x-pack/plugins/fleet/server/saved_objects/index.ts
View file

@ -156,6 +156,8 @@ const getSavedObjectTypes = (
revision: { type: 'integer' },
monitoring_enabled: { type: 'keyword', index: false },
is_preconfigured: { type: 'keyword' },
data_output_id: { type: 'keyword' },
monitoring_output_id: { type: 'keyword' },
},
},
migrations: {
@ -196,6 +198,7 @@ const getSavedObjectTypes = (
},
mappings: {
properties: {
output_id: { type: 'keyword', index: false },
name: { type: 'keyword' },
type: { type: 'keyword' },
is_default: { type: 'boolean' },
@ -203,6 +206,7 @@ const getSavedObjectTypes = (
ca_sha256: { type: 'keyword', index: false },
config: { type: 'flattened' },
config_yaml: { type: 'text' },
is_preconfigured: { type: 'boolean', index: false },
},
},
migrations: {

292
x-pack/plugins/fleet/server/services/agent_policies/__snapshots__/full_agent_policy.test.ts.snap generated Normal file
View file

@ -0,0 +1,292 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`getFullAgentPolicy should support a different data output 1`] = `
Object {
"agent": Object {
"monitoring": Object {
"enabled": true,
"logs": false,
"metrics": true,
"namespace": "default",
"use_output": "default",
},
},
"fleet": Object {
"hosts": Array [
"http://fleetserver:8220",
],
},
"id": "agent-policy",
"inputs": Array [],
"output_permissions": Object {
"data-output-id": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
},
"_fallback": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"logs-*",
"metrics-*",
"traces-*",
"synthetics-*",
".logs-endpoint.diagnostic.collection-*",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
"default": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"metrics-elastic_agent-default",
"metrics-elastic_agent.elastic_agent-default",
"metrics-elastic_agent.apm_server-default",
"metrics-elastic_agent.filebeat-default",
"metrics-elastic_agent.fleet_server-default",
"metrics-elastic_agent.metricbeat-default",
"metrics-elastic_agent.osquerybeat-default",
"metrics-elastic_agent.packetbeat-default",
"metrics-elastic_agent.endpoint_security-default",
"metrics-elastic_agent.auditbeat-default",
"metrics-elastic_agent.heartbeat-default",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
},
"outputs": Object {
"data-output-id": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://es-data.co:9201",
],
"type": "elasticsearch",
},
"default": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://127.0.0.1:9201",
],
"type": "elasticsearch",
},
},
"revision": 1,
}
`;
exports[`getFullAgentPolicy should support a different monitoring output 1`] = `
Object {
"agent": Object {
"monitoring": Object {
"enabled": true,
"logs": false,
"metrics": true,
"namespace": "default",
"use_output": "monitoring-output-id",
},
},
"fleet": Object {
"hosts": Array [
"http://fleetserver:8220",
],
},
"id": "agent-policy",
"inputs": Array [],
"output_permissions": Object {
"default": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
},
"_fallback": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"logs-*",
"metrics-*",
"traces-*",
"synthetics-*",
".logs-endpoint.diagnostic.collection-*",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
"monitoring-output-id": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"metrics-elastic_agent-default",
"metrics-elastic_agent.elastic_agent-default",
"metrics-elastic_agent.apm_server-default",
"metrics-elastic_agent.filebeat-default",
"metrics-elastic_agent.fleet_server-default",
"metrics-elastic_agent.metricbeat-default",
"metrics-elastic_agent.osquerybeat-default",
"metrics-elastic_agent.packetbeat-default",
"metrics-elastic_agent.endpoint_security-default",
"metrics-elastic_agent.auditbeat-default",
"metrics-elastic_agent.heartbeat-default",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
},
"outputs": Object {
"default": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://127.0.0.1:9201",
],
"type": "elasticsearch",
},
"monitoring-output-id": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://es-monitoring.co:9201",
],
"type": "elasticsearch",
},
},
"revision": 1,
}
`;
exports[`getFullAgentPolicy should support both different outputs for data and monitoring 1`] = `
Object {
"agent": Object {
"monitoring": Object {
"enabled": true,
"logs": false,
"metrics": true,
"namespace": "default",
"use_output": "monitoring-output-id",
},
},
"fleet": Object {
"hosts": Array [
"http://fleetserver:8220",
],
},
"id": "agent-policy",
"inputs": Array [],
"output_permissions": Object {
"data-output-id": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
},
"_fallback": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"logs-*",
"metrics-*",
"traces-*",
"synthetics-*",
".logs-endpoint.diagnostic.collection-*",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
"monitoring-output-id": Object {
"_elastic_agent_checks": Object {
"cluster": Array [
"monitor",
],
"indices": Array [
Object {
"names": Array [
"metrics-elastic_agent-default",
"metrics-elastic_agent.elastic_agent-default",
"metrics-elastic_agent.apm_server-default",
"metrics-elastic_agent.filebeat-default",
"metrics-elastic_agent.fleet_server-default",
"metrics-elastic_agent.metricbeat-default",
"metrics-elastic_agent.osquerybeat-default",
"metrics-elastic_agent.packetbeat-default",
"metrics-elastic_agent.endpoint_security-default",
"metrics-elastic_agent.auditbeat-default",
"metrics-elastic_agent.heartbeat-default",
],
"privileges": Array [
"auto_configure",
"create_doc",
],
},
],
},
},
},
"outputs": Object {
"data-output-id": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://es-data.co:9201",
],
"type": "elasticsearch",
},
"monitoring-output-id": Object {
"api_key": undefined,
"ca_sha256": undefined,
"hosts": Array [
"http://es-monitoring.co:9201",
],
"type": "elasticsearch",
},
},
"revision": 1,
}
`;

256
x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.test.ts Normal file
View file

@ -0,0 +1,256 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { savedObjectsClientMock } from 'src/core/server/mocks';
import type { AgentPolicy, Output } from '../../types';
import { agentPolicyService } from '../agent_policy';
import { agentPolicyUpdateEventHandler } from '../agent_policy_update';
import { getFullAgentPolicy } from './full_agent_policy';
const mockedAgentPolicyService = agentPolicyService as jest.Mocked<typeof agentPolicyService>;
function mockAgentPolicy(data: Partial<AgentPolicy>) {
mockedAgentPolicyService.get.mockResolvedValue({
id: 'agent-policy',
status: 'active',
package_policies: [],
is_managed: false,
namespace: 'default',
revision: 1,
name: 'Policy',
updated_at: '2020-01-01',
updated_by: 'qwerty',
...data,
});
}
jest.mock('../settings', () => {
return {
getSettings: () => {
return {
id: '93f74c0-e876-11ea-b7d3-8b2acec6f75c',
fleet_server_hosts: ['http://fleetserver:8220'],
};
},
};
});
jest.mock('../agent_policy');
jest.mock('../output', () => {
return {
outputService: {
getDefaultOutputId: () => 'test-id',
get: (soClient: any, id: string): Output => {
switch (id) {
case 'data-output-id':
return {
id: 'data-output-id',
is_default: false,
name: 'Data output',
// @ts-ignore
type: 'elasticsearch',
hosts: ['http://es-data.co:9201'],
};
case 'monitoring-output-id':
return {
id: 'monitoring-output-id',
is_default: false,
name: 'Monitoring output',
// @ts-ignore
type: 'elasticsearch',
hosts: ['http://es-monitoring.co:9201'],
};
default:
return {
id: 'test-id',
is_default: true,
name: 'default',
// @ts-ignore
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
};
}
},
},
};
});
jest.mock('../agent_policy_update');
jest.mock('../agents');
jest.mock('../package_policy');
function getAgentPolicyUpdateMock() {
return agentPolicyUpdateEventHandler as unknown as jest.Mock<
typeof agentPolicyUpdateEventHandler
>;
}
describe('getFullAgentPolicy', () => {
beforeEach(() => {
getAgentPolicyUpdateMock().mockClear();
mockedAgentPolicyService.get.mockReset();
});
it('should return a policy without monitoring if monitoring is not enabled', async () => {
mockAgentPolicy({
revision: 1,
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
enabled: false,
logs: false,
metrics: false,
},
},
});
});
it('should return a policy with monitoring if monitoring is enabled for logs', async () => {
mockAgentPolicy({
namespace: 'default',
revision: 1,
monitoring_enabled: ['logs'],
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
namespace: 'default',
use_output: 'default',
enabled: true,
logs: true,
metrics: false,
},
},
});
});
it('should return a policy with monitoring if monitoring is enabled for metrics', async () => {
mockAgentPolicy({
namespace: 'default',
revision: 1,
monitoring_enabled: ['metrics'],
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
namespace: 'default',
use_output: 'default',
enabled: true,
logs: false,
metrics: true,
},
},
});
});
it('should support a different monitoring output', async () => {
mockAgentPolicy({
namespace: 'default',
revision: 1,
monitoring_enabled: ['metrics'],
monitoring_output_id: 'monitoring-output-id',
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchSnapshot();
});
it('should support a different data output', async () => {
mockAgentPolicy({
namespace: 'default',
revision: 1,
monitoring_enabled: ['metrics'],
data_output_id: 'data-output-id',
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchSnapshot();
});
it('should support both different outputs for data and monitoring ', async () => {
mockAgentPolicy({
namespace: 'default',
revision: 1,
monitoring_enabled: ['metrics'],
data_output_id: 'data-output-id',
monitoring_output_id: 'monitoring-output-id',
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy).toMatchSnapshot();
});
it('should use "default" as the default policy id', async () => {
mockAgentPolicy({
id: 'policy',
status: 'active',
package_policies: [],
is_managed: false,
namespace: 'default',
revision: 1,
data_output_id: 'test-id',
monitoring_output_id: 'test-id',
});
const agentPolicy = await getFullAgentPolicy(savedObjectsClientMock.create(), 'agent-policy');
expect(agentPolicy?.outputs.default).toBeDefined();
});
});

229
x-pack/plugins/fleet/server/services/agent_policies/full_agent_policy.ts Normal file
View file

@ -0,0 +1,229 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import type { SavedObjectsClientContract } from 'kibana/server';
import { safeLoad } from 'js-yaml';
import type {
FullAgentPolicy,
PackagePolicy,
Settings,
Output,
FullAgentPolicyOutput,
} from '../../types';
import { agentPolicyService } from '../agent_policy';
import { outputService } from '../output';
import {
storedPackagePoliciesToAgentPermissions,
DEFAULT_PERMISSIONS,
} from '../package_policies_to_agent_permissions';
import { storedPackagePoliciesToAgentInputs, dataTypes, outputType } from '../../../common';
import type { FullAgentPolicyOutputPermissions } from '../../../common';
import { getSettings } from '../settings';
import { PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES, DEFAULT_OUTPUT } from '../../constants';
const MONITORING_DATASETS = [
'elastic_agent',
'elastic_agent.elastic_agent',
'elastic_agent.apm_server',
'elastic_agent.filebeat',
'elastic_agent.fleet_server',
'elastic_agent.metricbeat',
'elastic_agent.osquerybeat',
'elastic_agent.packetbeat',
'elastic_agent.endpoint_security',
'elastic_agent.auditbeat',
'elastic_agent.heartbeat',
];
export async function getFullAgentPolicy(
soClient: SavedObjectsClientContract,
id: string,
options?: { standalone: boolean }
): Promise<FullAgentPolicy | null> {
let agentPolicy;
const standalone = options?.standalone;
try {
agentPolicy = await agentPolicyService.get(soClient, id);
} catch (err) {
if (!err.isBoom || err.output.statusCode !== 404) {
throw err;
}
}
if (!agentPolicy) {
return null;
}
const defaultOutputId = await outputService.getDefaultOutputId(soClient);
if (!defaultOutputId) {
throw new Error('Default output is not setup');
}
const dataOutputId = agentPolicy.data_output_id || defaultOutputId;
const monitoringOutputId = agentPolicy.monitoring_output_id || defaultOutputId;
const outputs = await Promise.all(
Array.from(new Set([dataOutputId, monitoringOutputId])).map((outputId) =>
outputService.get(soClient, outputId)
)
);
const dataOutput = outputs.find((output) => output.id === dataOutputId);
if (!dataOutput) {
throw new Error(`Data output not found ${dataOutputId}`);
}
const monitoringOutput = outputs.find((output) => output.id === monitoringOutputId);
if (!monitoringOutput) {
throw new Error(`Monitoring output not found ${monitoringOutputId}`);
}
const fullAgentPolicy: FullAgentPolicy = {
id: agentPolicy.id,
outputs: {
...outputs.reduce<FullAgentPolicy['outputs']>((acc, output) => {
acc[getOutputIdForAgentPolicy(output)] = transformOutputToFullPolicyOutput(output);
return acc;
}, {}),
},
inputs: storedPackagePoliciesToAgentInputs(
agentPolicy.package_policies as PackagePolicy[],
getOutputIdForAgentPolicy(dataOutput)
),
revision: agentPolicy.revision,
...(agentPolicy.monitoring_enabled && agentPolicy.monitoring_enabled.length > 0
? {
agent: {
monitoring: {
namespace: agentPolicy.namespace,
use_output: getOutputIdForAgentPolicy(monitoringOutput),
enabled: true,
logs: agentPolicy.monitoring_enabled.includes(dataTypes.Logs),
metrics: agentPolicy.monitoring_enabled.includes(dataTypes.Metrics),
},
},
}
: {
agent: {
monitoring: { enabled: false, logs: false, metrics: false },
},
}),
};
const dataPermissions = (await storedPackagePoliciesToAgentPermissions(
soClient,
agentPolicy.package_policies
)) || { _fallback: DEFAULT_PERMISSIONS };
dataPermissions._elastic_agent_checks = {
cluster: DEFAULT_PERMISSIONS.cluster,
};
// TODO: fetch this from the elastic agent package https://github.com/elastic/kibana/issues/107738
const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace;
const monitoringPermissions: FullAgentPolicyOutputPermissions =
monitoringOutputId === dataOutputId
? dataPermissions
: {
_elastic_agent_checks: {
cluster: DEFAULT_PERMISSIONS.cluster,
},
};
if (
fullAgentPolicy.agent?.monitoring.enabled &&
monitoringNamespace &&
monitoringOutput &&
monitoringOutput.type === outputType.Elasticsearch
) {
let names: string[] = [];
if (fullAgentPolicy.agent.monitoring.logs) {
names = names.concat(
MONITORING_DATASETS.map((dataset) => `logs-${dataset}-${monitoringNamespace}`)
);
}
if (fullAgentPolicy.agent.monitoring.metrics) {
names = names.concat(
MONITORING_DATASETS.map((dataset) => `metrics-${dataset}-${monitoringNamespace}`)
);
}
monitoringPermissions._elastic_agent_checks.indices = [
{
names,
privileges: PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
},
];
}
// Only add permissions if output.type is "elasticsearch"
fullAgentPolicy.output_permissions = Object.keys(fullAgentPolicy.outputs).reduce<
NonNullable<FullAgentPolicy['output_permissions']>
>((outputPermissions, outputId) => {
const output = fullAgentPolicy.outputs[outputId];
if (output && output.type === outputType.Elasticsearch) {
outputPermissions[outputId] =
outputId === getOutputIdForAgentPolicy(dataOutput)
? dataPermissions
: monitoringPermissions;
}
return outputPermissions;
}, {});
// only add settings if not in standalone
if (!standalone) {
let settings: Settings;
try {
settings = await getSettings(soClient);
} catch (error) {
throw new Error('Default settings is not setup');
}
if (settings.fleet_server_hosts && settings.fleet_server_hosts.length) {
fullAgentPolicy.fleet = {
hosts: settings.fleet_server_hosts,
};
}
}
return fullAgentPolicy;
}
function transformOutputToFullPolicyOutput(
output: Output,
standalone = false
): FullAgentPolicyOutput {
// eslint-disable-next-line @typescript-eslint/naming-convention
const { config_yaml, type, hosts, ca_sha256, api_key } = output;
const configJs = config_yaml ? safeLoad(config_yaml) : {};
const newOutput: FullAgentPolicyOutput = {
type,
hosts,
ca_sha256,
api_key,
...configJs,
};
if (standalone) {
delete newOutput.api_key;
newOutput.username = 'ES_USERNAME';
newOutput.password = 'ES_PASSWORD';
}
return newOutput;
}
/**
* Get id used in full agent policy (sent to the agents)
* we use "default" for the default policy to avoid breaking changes
*/
function getOutputIdForAgentPolicy(output: Output) {
if (output.is_default) {
return DEFAULT_OUTPUT.name;
}
return output.id;
}

8
x-pack/plugins/fleet/server/services/agent_policies/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export { getFullAgentPolicy } from './full_agent_policy';

121
x-pack/plugins/fleet/server/services/agent_policy.test.ts
View file

@ -7,7 +7,7 @@
import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
import type { AgentPolicy, NewAgentPolicy, Output } from '../types';
import type { AgentPolicy, NewAgentPolicy } from '../types';
import { agentPolicyService } from './agent_policy';
import { agentPolicyUpdateEventHandler } from './agent_policy_update';
@ -47,24 +47,6 @@ function getSavedObjectMock(agentPolicyAttributes: any) {
return mock;
}
jest.mock('./output', () => {
return {
outputService: {
getDefaultOutputId: () => 'test-id',
get: (): Output => {
return {
id: 'test-id',
is_default: true,
name: 'default',
// @ts-ignore
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
};
},
},
};
});
jest.mock('./agent_policy_update');
jest.mock('./agents');
jest.mock('./package_policy');
@ -186,106 +168,17 @@ describe('agent policy', () => {
});
});
describe('getFullAgentPolicy', () => {
it('should return a policy without monitoring if monitoring is not enabled', async () => {
describe('bumpAllAgentPoliciesForOutput', () => {
it('should call agentPolicyUpdateEventHandler with updated event once', async () => {
const soClient = getSavedObjectMock({
revision: 1,
});
const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy');
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
enabled: false,
logs: false,
metrics: false,
},
},
});
});
it('should return a policy with monitoring if monitoring is enabled for logs', async () => {
const soClient = getSavedObjectMock({
namespace: 'default',
revision: 1,
monitoring_enabled: ['logs'],
});
const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy');
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
namespace: 'default',
use_output: 'default',
enabled: true,
logs: true,
metrics: false,
},
},
});
});
it('should return a policy with monitoring if monitoring is enabled for metrics', async () => {
const soClient = getSavedObjectMock({
namespace: 'default',
revision: 1,
monitoring_enabled: ['metrics'],
});
const agentPolicy = await agentPolicyService.getFullAgentPolicy(soClient, 'agent-policy');
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
expect(agentPolicy).toMatchObject({
id: 'agent-policy',
outputs: {
default: {
type: 'elasticsearch',
hosts: ['http://127.0.0.1:9201'],
ca_sha256: undefined,
api_key: undefined,
},
},
inputs: [],
revision: 1,
fleet: {
hosts: ['http://fleetserver:8220'],
},
agent: {
monitoring: {
namespace: 'default',
use_output: 'default',
enabled: true,
logs: false,
metrics: true,
},
},
});
await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, 'output-id-123');
expect(agentPolicyUpdateEventHandler).toHaveBeenCalledTimes(1);
});
});

197
x-pack/plugins/fleet/server/services/agent_policy.ts
View file

@ -6,7 +6,6 @@
*/
import { uniq, omit } from 'lodash';
import { safeLoad } from 'js-yaml';
import uuid from 'uuid/v4';
import type {
ElasticsearchClient,
@ -21,7 +20,6 @@ import {
AGENT_POLICY_SAVED_OBJECT_TYPE,
AGENT_SAVED_OBJECT_TYPE,
PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE,
PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
} from '../constants';
import type {
PackagePolicy,
@ -33,52 +31,27 @@ import type {
ListWithKuery,
NewPackagePolicy,
} from '../types';
import {
agentPolicyStatuses,
storedPackagePoliciesToAgentInputs,
dataTypes,
packageToPackagePolicy,
AGENT_POLICY_INDEX,
} from '../../common';
import { agentPolicyStatuses, packageToPackagePolicy, AGENT_POLICY_INDEX } from '../../common';
import type {
DeleteAgentPolicyResponse,
Settings,
FleetServerPolicy,
Installation,
Output,
DeletePackagePoliciesResponse,
} from '../../common';
import { AgentPolicyNameExistsError, HostedAgentPolicyRestrictionRelatedError } from '../errors';
import {
storedPackagePoliciesToAgentPermissions,
DEFAULT_PERMISSIONS,
} from '../services/package_policies_to_agent_permissions';
import { getPackageInfo } from './epm/packages';
import { getAgentsByKuery } from './agents';
import { packagePolicyService } from './package_policy';
import { outputService } from './output';
import { agentPolicyUpdateEventHandler } from './agent_policy_update';
import { getSettings } from './settings';
import { normalizeKuery, escapeSearchQueryPhrase } from './saved_object';
import { appContextService } from './app_context';
import { getFullAgentPolicy } from './agent_policies';
const SAVED_OBJECT_TYPE = AGENT_POLICY_SAVED_OBJECT_TYPE;
const MONITORING_DATASETS = [
'elastic_agent',
'elastic_agent.elastic_agent',
'elastic_agent.apm_server',
'elastic_agent.filebeat',
'elastic_agent.fleet_server',
'elastic_agent.metricbeat',
'elastic_agent.osquerybeat',
'elastic_agent.packetbeat',
'elastic_agent.endpoint_security',
'elastic_agent.auditbeat',
'elastic_agent.heartbeat',
];
class AgentPolicyService {
private triggerAgentPolicyUpdatedEvent = async (
soClient: SavedObjectsClientContract,
@ -472,6 +445,38 @@ class AgentPolicyService {
return res;
}
public async bumpAllAgentPoliciesForOutput(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
outputId: string,
options?: { user?: AuthenticatedUser }
): Promise<SavedObjectsBulkUpdateResponse<AgentPolicy>> {
const currentPolicies = await soClient.find<AgentPolicySOAttributes>({
type: SAVED_OBJECT_TYPE,
fields: ['revision', 'data_output_id', 'monitoring_output_id'],
searchFields: ['data_output_id', 'monitoring_output_id'],
search: escapeSearchQueryPhrase(outputId),
});
const bumpedPolicies = currentPolicies.saved_objects.map((policy) => {
policy.attributes = {
...policy.attributes,
revision: policy.attributes.revision + 1,
updated_at: new Date().toISOString(),
updated_by: options?.user ? options.user.username : 'system',
};
return policy;
});
const res = await soClient.bulkUpdate<AgentPolicySOAttributes>(bumpedPolicies);
await Promise.all(
currentPolicies.saved_objects.map((policy) =>
this.triggerAgentPolicyUpdatedEvent(soClient, esClient, 'updated', policy.id)
)
);
return res;
}
public async bumpAllAgentPolicies(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
@ -724,139 +729,7 @@ class AgentPolicyService {
id: string,
options?: { standalone: boolean }
): Promise<FullAgentPolicy | null> {
let agentPolicy;
const standalone = options?.standalone;
try {
agentPolicy = await this.get(soClient, id);
} catch (err) {
if (!err.isBoom || err.output.statusCode !== 404) {
throw err;
}
}
if (!agentPolicy) {
return null;
}
const defaultOutputId = await outputService.getDefaultOutputId(soClient);
if (!defaultOutputId) {
throw new Error('Default output is not setup');
}
const defaultOutput = await outputService.get(soClient, defaultOutputId);
const fullAgentPolicy: FullAgentPolicy = {
id: agentPolicy.id,
outputs: {
// TEMPORARY as we only support a default output
...[defaultOutput].reduce<FullAgentPolicy['outputs']>(
// eslint-disable-next-line @typescript-eslint/naming-convention
(outputs, { config_yaml, name, type, hosts, ca_sha256, api_key }) => {
const configJs = config_yaml ? safeLoad(config_yaml) : {};
outputs[name] = {
type,
hosts,
ca_sha256,
api_key,
...configJs,
};
if (options?.standalone) {
delete outputs[name].api_key;
outputs[name].username = 'ES_USERNAME';
outputs[name].password = 'ES_PASSWORD';
}
return outputs;
},
{}
),
},
inputs: storedPackagePoliciesToAgentInputs(agentPolicy.package_policies as PackagePolicy[]),
revision: agentPolicy.revision,
...(agentPolicy.monitoring_enabled && agentPolicy.monitoring_enabled.length > 0
? {
agent: {
monitoring: {
namespace: agentPolicy.namespace,
use_output: defaultOutput.name,
enabled: true,
logs: agentPolicy.monitoring_enabled.includes(dataTypes.Logs),
metrics: agentPolicy.monitoring_enabled.includes(dataTypes.Metrics),
},
},
}
: {
agent: {
monitoring: { enabled: false, logs: false, metrics: false },
},
}),
};
const permissions = (await storedPackagePoliciesToAgentPermissions(
soClient,
agentPolicy.package_policies
)) || { _fallback: DEFAULT_PERMISSIONS };
permissions._elastic_agent_checks = {
cluster: DEFAULT_PERMISSIONS.cluster,
};
// TODO: fetch this from the elastic agent package
const monitoringOutput = fullAgentPolicy.agent?.monitoring.use_output;
const monitoringNamespace = fullAgentPolicy.agent?.monitoring.namespace;
if (
fullAgentPolicy.agent?.monitoring.enabled &&
monitoringNamespace &&
monitoringOutput &&
fullAgentPolicy.outputs[monitoringOutput]?.type === 'elasticsearch'
) {
let names: string[] = [];
if (fullAgentPolicy.agent.monitoring.logs) {
names = names.concat(
MONITORING_DATASETS.map((dataset) => `logs-${dataset}-${monitoringNamespace}`)
);
}
if (fullAgentPolicy.agent.monitoring.metrics) {
names = names.concat(
MONITORING_DATASETS.map((dataset) => `metrics-${dataset}-${monitoringNamespace}`)
);
}
permissions._elastic_agent_checks.indices = [
{
names,
privileges: PACKAGE_POLICY_DEFAULT_INDEX_PRIVILEGES,
},
];
}
// Only add permissions if output.type is "elasticsearch"
fullAgentPolicy.output_permissions = Object.keys(fullAgentPolicy.outputs).reduce<
NonNullable<FullAgentPolicy['output_permissions']>
>((outputPermissions, outputName) => {
const output = fullAgentPolicy.outputs[outputName];
if (output && output.type === 'elasticsearch') {
outputPermissions[outputName] = permissions;
}
return outputPermissions;
}, {});
// only add settings if not in standalone
if (!standalone) {
let settings: Settings;
try {
settings = await getSettings(soClient);
} catch (error) {
throw new Error('Default settings is not setup');
}
if (settings.fleet_server_hosts && settings.fleet_server_hosts.length) {
fullAgentPolicy.fleet = {
hosts: settings.fleet_server_hosts,
};
}
}
return fullAgentPolicy;
return getFullAgentPolicy(soClient, id, options);
}
}

94
x-pack/plugins/fleet/server/services/output.test.ts
View file

@ -5,8 +5,10 @@
* 2.0.
*/
import { outputService } from './output';
import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
import type { OutputSOAttributes } from '../types';
import { outputService, outputIdToUuid } from './output';
import { appContextService } from './app_context';
jest.mock('./app_context');
@ -34,7 +36,97 @@ const CONFIG_WITHOUT_ES_HOSTS = {
},
};
function getMockedSoClient() {
const soClient = savedObjectsClientMock.create();
soClient.get.mockImplementation(async (type: string, id: string) => {
switch (id) {
case outputIdToUuid('output-test'): {
return {
id: outputIdToUuid('output-test'),
type: 'ingest-outputs',
references: [],
attributes: {
output_id: 'output-test',
},
};
}
default:
throw new Error('not found');
}
});
return soClient;
}
describe('Output Service', () => {
describe('create', () => {
it('work with a predefined id', async () => {
const soClient = getMockedSoClient();
soClient.create.mockResolvedValue({
id: outputIdToUuid('output-test'),
type: 'ingest-output',
attributes: {},
references: [],
});
await outputService.create(
soClient,
{
is_default: false,
name: 'Test',
type: 'elasticsearch',
},
{ id: 'output-test' }
);
expect(soClient.create).toBeCalled();
// ID should always be the same for a predefined id
expect(soClient.create.mock.calls[0][2]?.id).toEqual(outputIdToUuid('output-test'));
expect((soClient.create.mock.calls[0][1] as OutputSOAttributes).output_id).toEqual(
'output-test'
);
});
});
describe('get', () => {
it('work with a predefined id', async () => {
const soClient = getMockedSoClient();
const output = await outputService.get(soClient, 'output-test');
expect(soClient.get).toHaveBeenCalledWith('ingest-outputs', outputIdToUuid('output-test'));
expect(output.id).toEqual('output-test');
});
});
describe('getDefaultOutputId', () => {
it('work with a predefined id', async () => {
const soClient = getMockedSoClient();
soClient.find.mockResolvedValue({
page: 1,
per_page: 100,
total: 1,
saved_objects: [
{
id: outputIdToUuid('output-test'),
type: 'ingest-outputs',
references: [],
score: 0,
attributes: {
output_id: 'output-test',
is_default: true,
},
},
],
});
const defaultId = await outputService.getDefaultOutputId(soClient);
expect(soClient.find).toHaveBeenCalled();
expect(defaultId).toEqual('output-test');
});
});
describe('getDefaultESHosts', () => {
afterEach(() => {
mockedAppContextService.getConfig.mockReset();

116
x-pack/plugins/fleet/server/services/output.ts
View file

@ -5,7 +5,8 @@
* 2.0.
*/
import type { SavedObjectsClientContract } from 'src/core/server';
import type { SavedObject, SavedObjectsClientContract } from 'src/core/server';
import uuid from 'uuid/v5';
import type { NewOutput, Output, OutputSOAttributes } from '../types';
import { DEFAULT_OUTPUT, OUTPUT_SAVED_OBJECT_TYPE } from '../constants';
@ -17,8 +18,33 @@ const SAVED_OBJECT_TYPE = OUTPUT_SAVED_OBJECT_TYPE;
const DEFAULT_ES_HOSTS = ['http://localhost:9200'];
// differentiate
function isUUID(val: string) {
return (
typeof val === 'string' &&
val.match(/[0-9a-fA-F]{8}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{4}\-[0-9a-fA-F]{12}/)
);
}
export function outputIdToUuid(id: string) {
if (isUUID(id)) {
return id;
}
// UUID v5 need a namespace (uuid.DNS), changing this params will result in loosing the ability to generate predicable uuid
return uuid(id, uuid.DNS);
}
function outputSavedObjectToOutput(so: SavedObject<OutputSOAttributes>) {
const { output_id: outputId, ...atributes } = so.attributes;
return {
id: outputId ?? so.id,
...atributes,
};
}
class OutputService {
public async getDefaultOutput(soClient: SavedObjectsClientContract) {
private async _getDefaultOutputsSO(soClient: SavedObjectsClientContract) {
return await soClient.find<OutputSOAttributes>({
type: OUTPUT_SAVED_OBJECT_TYPE,
searchFields: ['is_default'],
@ -27,7 +53,7 @@ class OutputService {
}
public async ensureDefaultOutput(soClient: SavedObjectsClientContract) {
const outputs = await this.getDefaultOutput(soClient);
const outputs = await this._getDefaultOutputsSO(soClient);
if (!outputs.saved_objects.length) {
const newDefaultOutput = {
@ -39,10 +65,7 @@ class OutputService {
return await this.create(soClient, newDefaultOutput);
}
return {
id: outputs.saved_objects[0].id,
...outputs.saved_objects[0].attributes,
};
return outputSavedObjectToOutput(outputs.saved_objects[0]);
}
public getDefaultESHosts(): string[] {
@ -60,49 +83,84 @@ class OutputService {
}
public async getDefaultOutputId(soClient: SavedObjectsClientContract) {
const outputs = await this.getDefaultOutput(soClient);
const outputs = await this._getDefaultOutputsSO(soClient);
if (!outputs.saved_objects.length) {
return null;
}
return outputs.saved_objects[0].id;
return outputSavedObjectToOutput(outputs.saved_objects[0]).id;
}
public async create(
soClient: SavedObjectsClientContract,
output: NewOutput,
options?: { id?: string }
options?: { id?: string; overwrite?: boolean }
): Promise<Output> {
const data = { ...output };
const data: OutputSOAttributes = { ...output };
// ensure only default output exists
if (data.is_default) {
const defaultOuput = await this.getDefaultOutputId(soClient);
if (defaultOuput) {
throw new Error(`A default output already exists (${defaultOuput})`);
}
}
if (data.hosts) {
data.hosts = data.hosts.map(normalizeHostsForAgents);
}
const newSo = await soClient.create<OutputSOAttributes>(
SAVED_OBJECT_TYPE,
data as Output,
options
);
if (options?.id) {
data.output_id = options?.id;
}
const newSo = await soClient.create<OutputSOAttributes>(SAVED_OBJECT_TYPE, data, {
...options,
id: options?.id ? outputIdToUuid(options.id) : undefined,
});
return {
id: newSo.id,
id: options?.id ?? newSo.id,
...newSo.attributes,
};
}
public async bulkGet(
soClient: SavedObjectsClientContract,
ids: string[],
{ ignoreNotFound = false } = { ignoreNotFound: true }
) {
const res = await soClient.bulkGet<OutputSOAttributes>(
ids.map((id) => ({ id: outputIdToUuid(id), type: SAVED_OBJECT_TYPE }))
);
return res.saved_objects
.map((so) => {
if (so.error) {
if (!ignoreNotFound || so.error.statusCode !== 404) {
throw so.error;
}
return undefined;
}
return outputSavedObjectToOutput(so);
})
.filter((output): output is Output => typeof output !== 'undefined');
}
public async get(soClient: SavedObjectsClientContract, id: string): Promise<Output> {
const outputSO = await soClient.get<OutputSOAttributes>(SAVED_OBJECT_TYPE, id);
const outputSO = await soClient.get<OutputSOAttributes>(SAVED_OBJECT_TYPE, outputIdToUuid(id));
if (outputSO.error) {
throw new Error(outputSO.error.message);
}
return {
id: outputSO.id,
...outputSO.attributes,
};
return outputSavedObjectToOutput(outputSO);
}
public async delete(soClient: SavedObjectsClientContract, id: string) {
return soClient.delete(SAVED_OBJECT_TYPE, outputIdToUuid(id));
}
public async update(soClient: SavedObjectsClientContract, id: string, data: Partial<Output>) {
@ -111,8 +169,11 @@ class OutputService {
if (updateData.hosts) {
updateData.hosts = updateData.hosts.map(normalizeHostsForAgents);
}
const outputSO = await soClient.update<OutputSOAttributes>(SAVED_OBJECT_TYPE, id, updateData);
const outputSO = await soClient.update<OutputSOAttributes>(
SAVED_OBJECT_TYPE,
outputIdToUuid(id),
updateData
);
if (outputSO.error) {
throw new Error(outputSO.error.message);
@ -127,12 +188,7 @@ class OutputService {
});
return {
items: outputs.saved_objects.map<Output>((outputSO) => {
return {
id: outputSO.id,
...outputSO.attributes,
};
}),
items: outputs.saved_objects.map<Output>(outputSavedObjectToOutput),
total: outputs.total,
page: 1,
perPage: 1000,

178
x-pack/plugins/fleet/server/services/preconfiguration.test.ts
View file

@ -9,7 +9,7 @@ import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/serve
import { SavedObjectsErrorHelpers } from '../../../../../src/core/server';
import type { PreconfiguredAgentPolicy } from '../../common/types';
import type { PreconfiguredAgentPolicy, PreconfiguredOutput } from '../../common/types';
import type { AgentPolicy, NewPackagePolicy, Output } from '../types';
import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../constants';
@ -19,9 +19,15 @@ import * as agentPolicy from './agent_policy';
import {
ensurePreconfiguredPackagesAndPolicies,
comparePreconfiguredPolicyToCurrent,
ensurePreconfiguredOutputs,
cleanPreconfiguredOutputs,
} from './preconfiguration';
import { outputService } from './output';
jest.mock('./agent_policy_update');
jest.mock('./output');
const mockedOutputService = outputService as jest.Mocked<typeof outputService>;
const mockInstalledPackages = new Map();
const mockConfiguredPolicies = new Map();
@ -156,12 +162,17 @@ jest.mock('./app_context', () => ({
}));
const spyAgentPolicyServiceUpdate = jest.spyOn(agentPolicy.agentPolicyService, 'update');
const spyAgentPolicyServicBumpAllAgentPoliciesForOutput = jest.spyOn(
agentPolicy.agentPolicyService,
'bumpAllAgentPoliciesForOutput'
);
describe('policy preconfiguration', () => {
beforeEach(() => {
mockInstalledPackages.clear();
mockConfiguredPolicies.clear();
spyAgentPolicyServiceUpdate.mockClear();
spyAgentPolicyServicBumpAllAgentPoliciesForOutput.mockClear();
});
it('should perform a no-op when passed no policies or packages', async () => {
@ -480,3 +491,168 @@ describe('comparePreconfiguredPolicyToCurrent', () => {
expect(hasChanged).toBe(false);
});
});
describe('output preconfiguration', () => {
beforeEach(() => {
mockedOutputService.create.mockReset();
mockedOutputService.update.mockReset();
mockedOutputService.getDefaultESHosts.mockReturnValue(['http://default-es:9200']);
mockedOutputService.bulkGet.mockImplementation(async (soClient, id): Promise<Output[]> => {
return [
{
id: 'existing-output-1',
is_default: false,
name: 'Output 1',
// @ts-ignore
type: 'elasticsearch',
hosts: ['http://es.co:80'],
is_preconfigured: true,
},
];
});
});
it('should create preconfigured output that does not exists', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
await ensurePreconfiguredOutputs(soClient, esClient, [
{
id: 'non-existing-output-1',
name: 'Output 1',
type: 'elasticsearch',
is_default: false,
hosts: ['http://test.fr'],
},
]);
expect(mockedOutputService.create).toBeCalled();
expect(mockedOutputService.update).not.toBeCalled();
expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).not.toBeCalled();
});
it('should set default hosts if hosts is not set output that does not exists', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
await ensurePreconfiguredOutputs(soClient, esClient, [
{
id: 'non-existing-output-1',
name: 'Output 1',
type: 'elasticsearch',
is_default: false,
},
]);
expect(mockedOutputService.create).toBeCalled();
expect(mockedOutputService.create.mock.calls[0][1].hosts).toEqual(['http://default-es:9200']);
});
it('should update output if preconfigured output exists and changed', async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
soClient.find.mockResolvedValue({ saved_objects: [], page: 0, per_page: 0, total: 0 });
await ensurePreconfiguredOutputs(soClient, esClient, [
{
id: 'existing-output-1',
is_default: false,
name: 'Output 1',
type: 'elasticsearch',
hosts: ['http://newhostichanged.co:9201'], // field that changed
},
]);
expect(mockedOutputService.create).not.toBeCalled();
expect(mockedOutputService.update).toBeCalled();
expect(spyAgentPolicyServicBumpAllAgentPoliciesForOutput).toBeCalled();
});
const SCENARIOS: Array<{ name: string; data: PreconfiguredOutput }> = [
{
name: 'no changes',
data: {
id: 'existing-output-1',
is_default: false,
name: 'Output 1',
type: 'elasticsearch',
hosts: ['http://es.co:80'],
},
},
{
name: 'hosts without port',
data: {
id: 'existing-output-1',
is_default: false,
name: 'Output 1',
type: 'elasticsearch',
hosts: ['http://es.co'],
},
},
];
SCENARIOS.forEach((scenario) => {
const { data, name } = scenario;
it(`should do nothing if preconfigured output exists and did not changed (${name})`, async () => {
const soClient = savedObjectsClientMock.create();
const esClient = elasticsearchServiceMock.createClusterClient().asInternalUser;
await ensurePreconfiguredOutputs(soClient, esClient, [data]);
expect(mockedOutputService.create).not.toBeCalled();
expect(mockedOutputService.update).not.toBeCalled();
});
});
it('should not delete non deleted preconfigured output', async () => {
const soClient = savedObjectsClientMock.create();
mockedOutputService.list.mockResolvedValue({
items: [
{ id: 'output1', is_preconfigured: true } as Output,
{ id: 'output2', is_preconfigured: true } as Output,
],
page: 1,
perPage: 10000,
total: 1,
});
await cleanPreconfiguredOutputs(soClient, [
{
id: 'output1',
is_default: false,
name: 'Output 1',
type: 'elasticsearch',
hosts: ['http://es.co:9201'],
},
{
id: 'output2',
is_default: false,
name: 'Output 2',
type: 'elasticsearch',
hosts: ['http://es.co:9201'],
},
]);
expect(mockedOutputService.delete).not.toBeCalled();
});
it('should delete deleted preconfigured output', async () => {
const soClient = savedObjectsClientMock.create();
mockedOutputService.list.mockResolvedValue({
items: [
{ id: 'output1', is_preconfigured: true } as Output,
{ id: 'output2', is_preconfigured: true } as Output,
],
page: 1,
perPage: 10000,
total: 1,
});
await cleanPreconfiguredOutputs(soClient, [
{
id: 'output1',
is_default: false,
name: 'Output 1',
type: 'elasticsearch',
hosts: ['http://es.co:9201'],
},
]);
expect(mockedOutputService.delete).toBeCalled();
expect(mockedOutputService.delete).toBeCalledTimes(1);
expect(mockedOutputService.delete.mock.calls[0][1]).toEqual('output2');
});
});

92
x-pack/plugins/fleet/server/services/preconfiguration.ts
View file

@ -8,6 +8,7 @@
import type { ElasticsearchClient, SavedObjectsClientContract } from 'src/core/server';
import { i18n } from '@kbn/i18n';
import { groupBy, omit, pick, isEqual } from 'lodash';
import { safeDump } from 'js-yaml';
import type {
NewPackagePolicy,
@ -17,16 +18,15 @@ import type {
PreconfiguredAgentPolicy,
PreconfiguredPackage,
PreconfigurationError,
PreconfiguredOutput,
} from '../../common';
import { AGENT_POLICY_SAVED_OBJECT_TYPE } from '../../common';
import { AGENT_POLICY_SAVED_OBJECT_TYPE, normalizeHostsForAgents } from '../../common';
import {
PRECONFIGURATION_DELETION_RECORD_SAVED_OBJECT_TYPE,
PRECONFIGURATION_LATEST_KEYWORD,
} from '../constants';
import { escapeSearchQueryPhrase } from './saved_object';
import { pkgToPkgKey } from './epm/registry';
import { getInstallation, getPackageInfo } from './epm/packages';
import { ensurePackagesCompletedInstall } from './epm/packages/install';
@ -35,6 +35,7 @@ import { agentPolicyService, addPackageToAgentPolicy } from './agent_policy';
import type { InputsOverride } from './package_policy';
import { overridePackageInputs } from './package_policy';
import { appContextService } from './app_context';
import { outputService } from './output';
interface PreconfigurationResult {
policies: Array<{ id: string; updated_at: string }>;
@ -42,6 +43,89 @@ interface PreconfigurationResult {
nonFatalErrors: PreconfigurationError[];
}
function isPreconfiguredOutputDifferentFromCurrent(
existingOutput: Output,
preconfiguredOutput: Partial<Output>
): boolean {
return (
existingOutput.is_default !== preconfiguredOutput.is_default ||
existingOutput.name !== preconfiguredOutput.name ||
existingOutput.type !== preconfiguredOutput.type ||
(preconfiguredOutput.hosts &&
!isEqual(
existingOutput.hosts?.map(normalizeHostsForAgents),
preconfiguredOutput.hosts.map(normalizeHostsForAgents)
)) ||
existingOutput.ca_sha256 !== preconfiguredOutput.ca_sha256 ||
existingOutput.config_yaml !== preconfiguredOutput.config_yaml
);
}
export async function ensurePreconfiguredOutputs(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
outputs: PreconfiguredOutput[]
) {
if (outputs.length === 0) {
return;
}
const existingOutputs = await outputService.bulkGet(
soClient,
outputs.map(({ id }) => id),
{ ignoreNotFound: true }
);
await Promise.all(
outputs.map(async (output) => {
const existingOutput = existingOutputs.find((o) => o.id === output.id);
const { id, config, ...outputData } = output;
const configYaml = config ? safeDump(config) : undefined;
const data = {
...outputData,
config_yaml: configYaml,
is_preconfigured: true,
};
if (!data.hosts || data.hosts.length === 0) {
data.hosts = outputService.getDefaultESHosts();
}
if (!existingOutput) {
await outputService.create(soClient, data, { id, overwrite: true });
} else if (isPreconfiguredOutputDifferentFromCurrent(existingOutput, data)) {
await outputService.update(soClient, id, data);
// Bump revision of all policies using that output
if (outputData.is_default) {
await agentPolicyService.bumpAllAgentPolicies(soClient, esClient);
} else {
await agentPolicyService.bumpAllAgentPoliciesForOutput(soClient, esClient, id);
}
}
})
);
}
export async function cleanPreconfiguredOutputs(
soClient: SavedObjectsClientContract,
outputs: PreconfiguredOutput[]
) {
const existingPreconfiguredOutput = (await outputService.list(soClient)).items.filter(
(o) => o.is_preconfigured === true
);
const logger = appContextService.getLogger();
for (const output of existingPreconfiguredOutput) {
if (!outputs.find(({ id }) => output.id === id)) {
logger.info(`Deleting preconfigured output ${output.id}`);
await outputService.delete(soClient, output.id);
}
}
}
export async function ensurePreconfiguredPackagesAndPolicies(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient,
@ -224,7 +308,7 @@ export async function ensurePreconfiguredPackagesAndPolicies(
}
// Add the is_managed flag after configuring package policies to avoid errors
if (shouldAddIsManagedFlag) {
agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true });
await agentPolicyService.update(soClient, esClient, policy!.id, { is_managed: true });
}
}
}

30
x-pack/plugins/fleet/server/services/setup.ts
View file

@ -15,7 +15,11 @@ import { SO_SEARCH_LIMIT, DEFAULT_PACKAGES } from '../constants';
import { appContextService } from './app_context';
import { agentPolicyService } from './agent_policy';
import { ensurePreconfiguredPackagesAndPolicies } from './preconfiguration';
import {
cleanPreconfiguredOutputs,
ensurePreconfiguredOutputs,
ensurePreconfiguredPackagesAndPolicies,
} from './preconfiguration';
import { outputService } from './output';
import { generateEnrollmentAPIKey, hasEnrollementAPIKeysForPolicy } from './api_keys';
@ -45,23 +49,27 @@ async function createSetupSideEffects(
soClient: SavedObjectsClientContract,
esClient: ElasticsearchClient
): Promise<SetupStatus> {
const [defaultOutput] = await Promise.all([
outputService.ensureDefaultOutput(soClient),
const {
agentPolicies: policiesOrUndefined,
packages: packagesOrUndefined,
outputs: outputsOrUndefined,
} = appContextService.getConfig() ?? {};
const policies = policiesOrUndefined ?? [];
let packages = packagesOrUndefined ?? [];
await Promise.all([
ensurePreconfiguredOutputs(soClient, esClient, outputsOrUndefined ?? []),
settingsService.settingsSetup(soClient),
]);
const defaultOutput = await outputService.ensureDefaultOutput(soClient);
await awaitIfFleetServerSetupPending();
if (appContextService.getConfig()?.agentIdVerificationEnabled) {
await ensureFleetGlobalEsAssets(soClient, esClient);
}
const { agentPolicies: policiesOrUndefined, packages: packagesOrUndefined } =
appContextService.getConfig() ?? {};
const policies = policiesOrUndefined ?? [];
let packages = packagesOrUndefined ?? [];
// Ensure that required packages are always installed even if they're left out of the config
const preconfiguredPackageNames = new Set(packages.map((pkg) => pkg.name));
@ -90,6 +98,8 @@ async function createSetupSideEffects(
defaultOutput
);
await cleanPreconfiguredOutputs(soClient, outputsOrUndefined ?? []);
await ensureDefaultEnrollmentAPIKeysExists(soClient, esClient);
await ensureAgentActionPolicyChangeExists(soClient, esClient);

1
x-pack/plugins/fleet/server/types/index.tsx
View file

@ -27,6 +27,7 @@ export {
PackagePolicySOAttributes,
FullAgentPolicyInput,
FullAgentPolicy,
FullAgentPolicyOutput,
AgentPolicy,
AgentPolicySOAttributes,
NewAgentPolicy,

81
x-pack/plugins/fleet/server/types/models/preconfiguration.test.ts Normal file
View file

@ -0,0 +1,81 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { PreconfiguredOutputsSchema, PreconfiguredAgentPoliciesSchema } from './preconfiguration';
describe('Test preconfiguration schema', () => {
describe('PreconfiguredOutputsSchema', () => {
it('should not allow multiple default output', () => {
expect(() => {
PreconfiguredOutputsSchema.validate([
{
id: 'output-1',
name: 'Output 1',
type: 'elasticsearch',
is_default: true,
},
{
id: 'output-2',
name: 'Output 2',
type: 'elasticsearch',
is_default: true,
},
]);
}).toThrowError('preconfigured outputs need to have only one default output.');
});
it('should not allow multiple output with same ids', () => {
expect(() => {
PreconfiguredOutputsSchema.validate([
{
id: 'nonuniqueid',
name: 'Output 1',
type: 'elasticsearch',
},
{
id: 'nonuniqueid',
name: 'Output 2',
type: 'elasticsearch',
},
]);
}).toThrowError('preconfigured outputs need to have unique ids.');
});
it('should not allow multiple output with same names', () => {
expect(() => {
PreconfiguredOutputsSchema.validate([
{
id: 'output-1',
name: 'nonuniquename',
type: 'elasticsearch',
},
{
id: 'output-2',
name: 'nonuniquename',
type: 'elasticsearch',
},
]);
}).toThrowError('preconfigured outputs need to have unique names.');
});
});
describe('PreconfiguredAgentPoliciesSchema', () => {
it('should not allow multiple outputs in one policy', () => {
expect(() => {
PreconfiguredAgentPoliciesSchema.validate([
{
id: 'policy-1',
name: 'Policy 1',
package_policies: [],
data_output_id: 'test1',
monitoring_output_id: 'test2',
},
]);
}).toThrowError(
'[0]: Currently Fleet only support one output per agent policy data_output_id should be the same as monitoring_output_id.'
);
});
});
});

127
x-pack/plugins/fleet/server/types/models/preconfiguration.ts
View file

@ -14,6 +14,8 @@ import {
DEFAULT_FLEET_SERVER_AGENT_POLICY,
DEFAULT_PACKAGES,
} from '../../constants';
import type { PreconfiguredOutput } from '../../../common';
import { outputType } from '../../../common';
import { AgentPolicyBaseSchema } from './agent_policy';
import { NamespaceSchema } from './package_policy';
@ -47,47 +49,94 @@ export const PreconfiguredPackagesSchema = schema.arrayOf(
}
);
export const PreconfiguredAgentPoliciesSchema = schema.arrayOf(
function validatePreconfiguredOutputs(outputs: PreconfiguredOutput[]) {
const acc = { names: new Set(), ids: new Set(), is_default: false };
for (const output of outputs) {
if (acc.names.has(output.name)) {
return 'preconfigured outputs need to have unique names.';
}
if (acc.ids.has(output.id)) {
return 'preconfigured outputs need to have unique ids.';
}
if (acc.is_default && output.is_default) {
return 'preconfigured outputs need to have only one default output.';
}
acc.ids.add(output.id);
acc.names.add(output.name);
acc.is_default = acc.is_default || output.is_default;
}
}
export const PreconfiguredOutputsSchema = schema.arrayOf(
schema.object({
...AgentPolicyBaseSchema,
namespace: schema.maybe(NamespaceSchema),
id: schema.maybe(schema.oneOf([schema.string(), schema.number()])),
is_default: schema.maybe(schema.boolean()),
is_default_fleet_server: schema.maybe(schema.boolean()),
package_policies: schema.arrayOf(
schema.object({
name: schema.string(),
package: schema.object({
name: schema.string(),
}),
description: schema.maybe(schema.string()),
namespace: schema.maybe(NamespaceSchema),
inputs: schema.maybe(
schema.arrayOf(
schema.object({
type: schema.string(),
enabled: schema.maybe(schema.boolean()),
keep_enabled: schema.maybe(schema.boolean()),
vars: varsSchema,
streams: schema.maybe(
schema.arrayOf(
schema.object({
data_stream: schema.object({
type: schema.maybe(schema.string()),
dataset: schema.string(),
}),
enabled: schema.maybe(schema.boolean()),
keep_enabled: schema.maybe(schema.boolean()),
vars: varsSchema,
})
)
),
})
)
),
})
),
id: schema.string(),
is_default: schema.boolean({ defaultValue: false }),
name: schema.string(),
type: schema.oneOf([schema.literal(outputType.Elasticsearch)]),
hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))),
ca_sha256: schema.maybe(schema.string()),
config: schema.maybe(schema.object({}, { unknowns: 'allow' })),
}),
{
defaultValue: [],
validate: validatePreconfiguredOutputs,
}
);
export const PreconfiguredAgentPoliciesSchema = schema.arrayOf(
schema.object(
{
...AgentPolicyBaseSchema,
namespace: schema.maybe(NamespaceSchema),
id: schema.maybe(schema.oneOf([schema.string(), schema.number()])),
is_default: schema.maybe(schema.boolean()),
is_default_fleet_server: schema.maybe(schema.boolean()),
data_output_id: schema.maybe(schema.string()),
monitoring_output_id: schema.maybe(schema.string()),
package_policies: schema.arrayOf(
schema.object({
name: schema.string(),
package: schema.object({
name: schema.string(),
}),
description: schema.maybe(schema.string()),
namespace: schema.maybe(NamespaceSchema),
inputs: schema.maybe(
schema.arrayOf(
schema.object({
type: schema.string(),
enabled: schema.maybe(schema.boolean()),
keep_enabled: schema.maybe(schema.boolean()),
vars: varsSchema,
streams: schema.maybe(
schema.arrayOf(
schema.object({
data_stream: schema.object({
type: schema.maybe(schema.string()),
dataset: schema.string(),
}),
enabled: schema.maybe(schema.boolean()),
keep_enabled: schema.maybe(schema.boolean()),
vars: varsSchema,
})
)
),
})
)
),
})
),
},
{
validate: (policy) => {
if (policy.data_output_id !== policy.monitoring_output_id) {
return 'Currently Fleet only support one output per agent policy data_output_id should be the same as monitoring_output_id.';
}
},
}
),
{
defaultValue: [DEFAULT_AGENT_POLICY, DEFAULT_FLEET_SERVER_AGENT_POLICY],
}