maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SECURITY_SOLUTION][ENDPOINT] Improve Endpoint Host data generator to also integrate with Ingest (#74305)

Browse source 
* Endpoint generator connects host with a real policy and enrolls agent

Co-authored-by: Paul Tavares <paul.tavares@elastic.co>
Co-authored-by: kevinlog <kevin.logan@elastic.co>
Co-authored-by: Candace Park <candace.park@elastic.co>
This commit is contained in:
Paul Tavares 2020-09-30 16:07:58 -04:00 committed by GitHub
parent df86dcb215
commit 15e7623ecf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 490 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/kbn-dev-utils/src/kbn_client/kbn_client.ts
View file

@ -54,8 +54,8 @@ export class KbnClient {
/**
* Make a direct request to the Kibana server
*/
async request(options: ReqOptions) {
return await this.requester.request(options);
async request<T>(options: ReqOptions) {
return await this.requester.request<T>(options);
}
resolveUrl(relativeUrl: string) {

24
x-pack/plugins/security_solution/common/endpoint/generate_data.ts
View file

@ -110,6 +110,12 @@ const Mac: OSFields[] = [];
const OS: OSFields[] = [...Windows, ...Mac, ...Linux];
const POLICY_RESPONSE_STATUSES: HostPolicyResponseActionStatus[] = [
HostPolicyResponseActionStatus.success,
HostPolicyResponseActionStatus.failure,
HostPolicyResponseActionStatus.warning,
];
const APPLIED_POLICIES: Array<{
name: string;
id: string;
@ -125,6 +131,11 @@ const APPLIED_POLICIES: Array<{
id: 'C2A9093E-E289-4C0A-AA44-8C32A414FA7A',
status: HostPolicyResponseActionStatus.success,
},
{
name: 'Detect Malware Only',
id: '47d7965d-6869-478b-bd9c-fb0d2bb3959f',
status: HostPolicyResponseActionStatus.success,
},
];
const FILE_OPERATIONS: string[] = ['creation', 'open', 'rename', 'execution', 'deletion'];
@ -364,15 +375,12 @@ export class EndpointDocGenerator {
}
/**
* Creates new random policy id for the host to simulate new policy application
* Updates the current Host common record applied Policy to a different one from the list
* of random choices and gives it a random policy response status.
*/
public updatePolicyId() {
this.commonInfo.Endpoint.policy.applied.id = this.randomChoice(APPLIED_POLICIES).id;
this.commonInfo.Endpoint.policy.applied.status = this.randomChoice([
HostPolicyResponseActionStatus.success,
HostPolicyResponseActionStatus.failure,
HostPolicyResponseActionStatus.warning,
]);
public updateHostPolicyData() {
this.commonInfo.Endpoint.policy.applied = this.randomChoice(APPLIED_POLICIES);
this.commonInfo.Endpoint.policy.applied.status = this.randomChoice(POLICY_RESPONSE_STATUSES);
}
private createHostData(): HostInfo {

393
x-pack/plugins/security_solution/common/endpoint/index_data.ts
View file

@ -6,25 +6,66 @@
import { Client } from '@elastic/elasticsearch';
import seedrandom from 'seedrandom';
import { KbnClient } from '@kbn/dev-utils';
import { AxiosResponse } from 'axios';
import { EndpointDocGenerator, TreeOptions, Event } from './generate_data';
import { firstNonNullValue } from './models/ecs_safety_helpers';
import {
CreateAgentPolicyRequest,
CreateAgentPolicyResponse,
CreatePackagePolicyRequest,
CreatePackagePolicyResponse,
GetPackagesResponse,
PostAgentEnrollRequest,
AGENT_API_ROUTES,
AGENT_POLICY_API_ROUTES,
EPM_API_ROUTES,
PACKAGE_POLICY_API_ROUTES,
ENROLLMENT_API_KEY_ROUTES,
GetEnrollmentAPIKeysResponse,
GetOneEnrollmentAPIKeyResponse,
PostAgentEnrollResponse,
PostAgentCheckinRequest,
PostAgentCheckinResponse,
PostAgentAcksResponse,
PostAgentAcksRequest,
} from '../../../ingest_manager/common';
import { factory as policyConfigFactory } from './models/policy_config';
import { HostMetadata } from './types';
import { KbnClientWithApiKeySupport } from '../../scripts/endpoint/kbn_client_with_api_key_support';
export async function indexHostsAndAlerts(
client: Client,
kbnClient: KbnClientWithApiKeySupport,
seed: string,
numHosts: number,
numDocs: number,
metadataIndex: string,
policyIndex: string,
policyResponseIndex: string,
eventIndex: string,
alertIndex: string,
alertsPerHost: number,
fleet: boolean,
options: TreeOptions = {}
) {
const random = seedrandom(seed);
const epmEndpointPackage = await getEndpointPackageInfo(kbnClient);
// Keep a map of host applied policy ids (fake) to real ingest package configs (policy record)
const realPolicies: Record<string, CreatePackagePolicyResponse['item']> = {};
for (let i = 0; i < numHosts; i++) {
const generator = new EndpointDocGenerator(random);
await indexHostDocs(numDocs, client, metadataIndex, policyIndex, generator);
await indexHostDocs(
numDocs,
client,
kbnClient,
realPolicies,
epmEndpointPackage,
metadataIndex,
policyResponseIndex,
fleet,
generator
);
await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options);
}
await client.indices.refresh({
@ -43,22 +84,78 @@ function delay(ms: number) {
async function indexHostDocs(
numDocs: number,
client: Client,
kbnClient: KbnClientWithApiKeySupport,
realPolicies: Record<string, CreatePackagePolicyResponse['item']>,
epmEndpointPackage: GetPackagesResponse['response'][0],
metadataIndex: string,
policyIndex: string,
policyResponseIndex: string,
enrollFleet: boolean,
generator: EndpointDocGenerator
) {
const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents
const timestamp = new Date().getTime();
let hostMetadata: HostMetadata;
let wasAgentEnrolled = false;
let enrolledAgent: undefined | PostAgentEnrollResponse['item'];
for (let j = 0; j < numDocs; j++) {
generator.updateHostData();
generator.updatePolicyId();
generator.updateHostPolicyData();
hostMetadata = generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1));
if (enrollFleet) {
const { id: appliedPolicyId, name: appliedPolicyName } = hostMetadata.Endpoint.policy.applied;
// If we don't yet have a "real" policy record, then create it now in ingest (package config)
if (!realPolicies[appliedPolicyId]) {
// eslint-disable-next-line require-atomic-updates
realPolicies[appliedPolicyId] = await createPolicy(
kbnClient,
appliedPolicyName,
epmEndpointPackage.version
);
}
// If we did not yet enroll an agent for this Host, do it now that we have good policy id
if (!wasAgentEnrolled) {
wasAgentEnrolled = true;
enrolledAgent = await fleetEnrollAgentForHost(
kbnClient,
hostMetadata!,
realPolicies[appliedPolicyId].policy_id
);
}
// Update the Host metadata record with the ID of the "real" policy along with the enrolled agent id
hostMetadata = {
...hostMetadata,
elastic: {
...hostMetadata.elastic,
agent: {
...hostMetadata.elastic.agent,
id: enrolledAgent?.id ?? hostMetadata.elastic.agent.id,
},
},
Endpoint: {
...hostMetadata.Endpoint,
policy: {
...hostMetadata.Endpoint.policy,
applied: {
...hostMetadata.Endpoint.policy.applied,
id: realPolicies[appliedPolicyId].id,
},
},
},
};
}
await client.index({
index: metadataIndex,
body: generator.generateHostMetadata(timestamp - timeBetweenDocs * (numDocs - j - 1)),
body: hostMetadata,
op_type: 'create',
});
await client.index({
index: policyIndex,
index: policyResponseIndex,
body: generator.generatePolicyResponse(timestamp - timeBetweenDocs * (numDocs - j - 1)),
op_type: 'create',
});
@ -98,3 +195,287 @@ async function indexAlerts(
await client.bulk({ body, refresh: true });
}
}
const createPolicy = async (
kbnClient: KbnClient,
policyName: string,
endpointPackageVersion: string
): Promise<CreatePackagePolicyResponse['item']> => {
// Create Agent Policy first
const newAgentPolicyData: CreateAgentPolicyRequest['body'] = {
name: `Policy for ${policyName}`,
description: '',
namespace: 'default',
};
let agentPolicy;
try {
agentPolicy = (await kbnClient.request({
path: AGENT_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newAgentPolicyData,
})) as AxiosResponse<CreateAgentPolicyResponse>;
} catch (error) {
throw new Error(`create policy ${error}`);
}
// Create Package Configuration
const newPackagePolicyData: CreatePackagePolicyRequest['body'] = {
name: policyName,
description: 'Protect the worlds data',
policy_id: agentPolicy.data.item.id,
enabled: true,
output_id: '',
inputs: [
{
type: 'endpoint',
enabled: true,
streams: [],
config: {
policy: {
value: policyConfigFactory(),
},
},
},
],
namespace: 'default',
package: {
name: 'endpoint',
title: 'endpoint',
version: endpointPackageVersion,
},
};
const packagePolicy = (await kbnClient.request({
path: PACKAGE_POLICY_API_ROUTES.CREATE_PATTERN,
method: 'POST',
body: newPackagePolicyData,
})) as AxiosResponse<CreatePackagePolicyResponse>;
return packagePolicy.data.item;
};
const getEndpointPackageInfo = async (
kbnClient: KbnClient
): Promise<GetPackagesResponse['response'][0]> => {
const endpointPackage = ((await kbnClient.request({
path: `${EPM_API_ROUTES.LIST_PATTERN}?category=security`,
method: 'GET',
})) as AxiosResponse<GetPackagesResponse>).data.response.find(
(epmPackage) => epmPackage.name === 'endpoint'
);
if (!endpointPackage) {
throw new Error('EPM Endpoint package was not found!');
}
return endpointPackage;
};
const fleetEnrollAgentForHost = async (
kbnClient: KbnClientWithApiKeySupport,
endpointHost: HostMetadata,
agentPolicyId: string
): Promise<undefined | PostAgentEnrollResponse['item']> => {
// Get Enrollement key for host's applied policy
const enrollmentApiKey = await kbnClient
.request<GetEnrollmentAPIKeysResponse>({
path: ENROLLMENT_API_KEY_ROUTES.LIST_PATTERN,
method: 'GET',
query: {
kuery: `fleet-enrollment-api-keys.policy_id:"${agentPolicyId}"`,
},
})
.then((apiKeysResponse) => {
const apiKey = apiKeysResponse.data.list[0];
if (!apiKey) {
return Promise.reject(
new Error(`no API enrollment key found for agent policy id ${agentPolicyId}`)
);
}
return kbnClient
.request<GetOneEnrollmentAPIKeyResponse>({
path: ENROLLMENT_API_KEY_ROUTES.INFO_PATTERN.replace('{keyId}', apiKey.id),
method: 'GET',
})
.catch((error) => {
// eslint-disable-next-line no-console
console.log('unable to retrieve enrollment api key for policy');
return Promise.reject(error);
});
})
.then((apiKeyDetailsResponse) => {
return apiKeyDetailsResponse.data.item.api_key;
})
.catch((error) => {
// eslint-disable-next-line no-console
console.error(error);
return '';
});
if (enrollmentApiKey.length === 0) {
return;
}
const fetchKibanaVersion = async () => {
const version = ((await kbnClient.request({
path: '/api/status',
method: 'GET',
})) as AxiosResponse).data.version.number;
if (!version) {
// eslint-disable-next-line no-console
console.log('failed to retrieve kibana version');
}
return version;
};
// Enroll an agent for the Host
const body: PostAgentEnrollRequest['body'] = {
type: 'PERMANENT',
metadata: {
local: {
elastic: {
agent: {
version: await fetchKibanaVersion(),
},
},
host: {
architecture: 'x86_64',
hostname: endpointHost.host,
name: endpointHost.host,
id: '1c032ec0-3a94-4d54-9ad2-c5610c0eaba4',
ip: ['fe80::703b:b9e6:887d:7f5/64', '10.0.2.15/24', '::1/128', '127.0.0.1/8'],
mac: ['08:00:27:d8:c5:c0'],
},
os: {
family: 'windows',
kernel: '10.0.19041.388 (WinBuild.160101.0800)',
platform: 'windows',
version: '10.0',
name: 'Windows 10 Pro',
full: 'Windows 10 Pro(10.0)',
},
},
user_provided: {
dev_agent_version: '0.0.1',
region: 'us-east',
},
},
};
try {
// First enroll the agent
const res = await kbnClient.requestWithApiKey(AGENT_API_ROUTES.ENROLL_PATTERN, {
method: 'POST',
body: JSON.stringify(body),
headers: {
'kbn-xsrf': 'xxx',
Authorization: `ApiKey ${enrollmentApiKey}`,
'Content-Type': 'application/json',
},
});
if (res) {
const enrollObj: PostAgentEnrollResponse = await res.json();
if (!res.ok) {
// eslint-disable-next-line no-console
console.error('unable to enroll agent', enrollObj);
return;
}
// ------------------------------------------------
// now check the agent in so that it can complete enrollment
const checkinBody: PostAgentCheckinRequest['body'] = {
events: [
{
type: 'STATE',
subtype: 'RUNNING',
message: 'state changed from STOPPED to RUNNING',
timestamp: new Date().toISOString(),
payload: {
random: 'data',
state: 'RUNNING',
previous_state: 'STOPPED',
},
agent_id: enrollObj.item.id,
},
],
};
const checkinRes = await kbnClient
.requestWithApiKey(
AGENT_API_ROUTES.CHECKIN_PATTERN.replace('{agentId}', enrollObj.item.id),
{
method: 'POST',
body: JSON.stringify(checkinBody),
headers: {
'kbn-xsrf': 'xxx',
Authorization: `ApiKey ${enrollObj.item.access_api_key}`,
'Content-Type': 'application/json',
},
}
)
.catch((error) => {
return Promise.reject(error);
});
// Agent unenrolling?
if (checkinRes.status === 403) {
return;
}
const checkinObj: PostAgentCheckinResponse = await checkinRes.json();
if (!checkinRes.ok) {
// eslint-disable-next-line no-console
console.error(
`failed to checkin agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]`
);
return enrollObj.item;
}
// ------------------------------------------------
// If we have an action to ack(), then do it now
if (checkinObj.actions.length) {
const ackActionBody: PostAgentAcksRequest['body'] = {
// @ts-ignore
events: checkinObj.actions.map<PostAgentAcksRequest['body']['events'][0]>((action) => {
return {
action_id: action.id,
type: 'ACTION_RESULT',
subtype: 'CONFIG',
timestamp: new Date().toISOString(),
agent_id: action.agent_id,
policy_id: agentPolicyId,
message: `endpoint generator: Endpoint Started`,
};
}),
};
const ackActionResp = await kbnClient.requestWithApiKey(
AGENT_API_ROUTES.ACKS_PATTERN.replace('{agentId}', enrollObj.item.id),
{
method: 'POST',
body: JSON.stringify(ackActionBody),
headers: {
'kbn-xsrf': 'xxx',
Authorization: `ApiKey ${enrollObj.item.access_api_key}`,
'Content-Type': 'application/json',
},
}
);
const ackActionObj: PostAgentAcksResponse = await ackActionResp.json();
if (!ackActionResp.ok) {
// eslint-disable-next-line no-console
console.error(
`failed to ACK Actions provided to agent [${enrollObj.item.id}] for endpoint [${endpointHost.host.id}]`
);
// eslint-disable-next-line no-console
console.error(JSON.stringify(ackActionObj, null, 2));
return enrollObj.item;
}
}
return enrollObj.item;
}
} catch (error) {
// eslint-disable-next-line no-console
console.error(error);
}
};

32
x-pack/plugins/security_solution/scripts/endpoint/kbn_client_with_api_key_support.ts Normal file
View file

@ -0,0 +1,32 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { KibanaConfig } from '@kbn/dev-utils/target/kbn_client/kbn_client_requester';
import fetch, { RequestInit as FetchRequestInit } from 'node-fetch';
export class KbnClientWithApiKeySupport extends KbnClient {
private kibanaUrlNoAuth: string;
constructor(log: ToolingLog, kibanaConfig: KibanaConfig) {
super(log, kibanaConfig);
const kibanaUrl = this.resolveUrl(kibanaConfig.url);
const matches = kibanaUrl.match(/(https?:\/\/)(.*\:.*\@)(.*)/);
// strip auth from url
this.kibanaUrlNoAuth =
matches && matches.length >= 3
? matches[1] + matches[3].replace('/', '')
: kibanaUrl.replace('/', '');
}
/**
* The fleet api to enroll and agent requires an api key when you mke the request, however KbnClient currently does not support sending an api key with the request. This function allows you to send an api key with a request.
*/
requestWithApiKey(path: string, init?: RequestInit | undefined): Promise<Response> {
return (fetch(
`${this.kibanaUrlNoAuth}${path}`,
init as FetchRequestInit
) as unknown) as Promise<Response>;
}
}

93
x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts
View file

@ -4,14 +4,19 @@
* you may not use this file except in compliance with the Elastic License.
*/
/* eslint-disable no-console */
import * as path from 'path';
import yargs from 'yargs';
import * as url from 'url';
import fetch from 'node-fetch';
import { Client, ClientOptions } from '@elastic/elasticsearch';
import { ResponseError } from '@elastic/elasticsearch/lib/errors';
import { KbnClient, ToolingLog } from '@kbn/dev-utils';
import { AxiosResponse } from 'axios';
import { indexHostsAndAlerts } from '../../common/endpoint/index_data';
import { ANCESTRY_LIMIT } from '../../common/endpoint/generate_data';
import { FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../../ingest_manager/common/constants';
import {
CreateFleetSetupResponse,
PostIngestSetupResponse,
} from '../../../ingest_manager/common/types/rest_spec';
import { KbnClientWithApiKeySupport } from './kbn_client_with_api_key_support';
main();
@ -35,42 +40,37 @@ async function deleteIndices(indices: string[], client: Client) {
}
}
async function doIngestSetup(kibanaURL: string) {
async function doIngestSetup(kbnClient: KbnClient) {
// Setup Ingest
try {
const kbURL = new url.URL(kibanaURL);
// this includes the base path that is randomly generated by Kibana
const pathname = path.posix.join(path.posix.sep, kbURL.pathname, 'api/ingest_manager/setup');
const connectURL = new url.URL(pathname, kbURL);
console.log('Calling ingest manager setup at ', connectURL.toString());
const response = await fetch(
// wrap base url in URL class because the kibana basepath will get removed otherwise
connectURL.toString(),
{
method: 'POST',
headers: {
'kbn-xsrf': 'blah',
},
}
);
if (response.status !== 200) {
console.log('POST response ', response);
console.log(
'Request failed please check that you have the correct base path and credentials for the kibana URL'
);
// eslint-disable-next-line no-process-exit
process.exit(1);
}
const setupResponse = await response.json();
console.log('Ingest setup response ', setupResponse);
if (!setupResponse?.isInitialized) {
console.log('Initializing the ingest manager failed, existing');
// eslint-disable-next-line no-process-exit
process.exit(1);
const setupResponse = (await kbnClient.request({
path: SETUP_API_ROUTE,
method: 'POST',
})) as AxiosResponse<PostIngestSetupResponse>;
if (!setupResponse.data.isInitialized) {
console.error(setupResponse.data);
throw new Error('Initializing the ingest manager failed, existing');
}
} catch (error) {
console.log(JSON.stringify(error, null, 2));
// eslint-disable-next-line no-process-exit
process.exit(1);
console.error(error);
throw error;
}
// Setup Fleet
try {
const setupResponse = (await kbnClient.request({
path: FLEET_SETUP_API_ROUTES.CREATE_PATTERN,
method: 'POST',
})) as AxiosResponse<CreateFleetSetupResponse>;
if (!setupResponse.data.isInitialized) {
console.error(setupResponse.data);
throw new Error('Initializing Fleet failed, existing');
}
} catch (error) {
console.error(error);
throw error;
}
}
@ -196,14 +196,25 @@ async function main() {
type: 'boolean',
default: false,
},
fleet: {
alias: 'f',
describe: 'enroll fleet agents for hosts',
type: 'boolean',
default: false,
},
}).argv;
await doIngestSetup(argv.kibana);
const kbnClient = new KbnClientWithApiKeySupport(new ToolingLog(), { url: argv.kibana });
const clientOptions: ClientOptions = {
node: argv.node,
};
try {
await doIngestSetup(kbnClient);
} catch (error) {
// eslint-disable-next-line no-process-exit
process.exit(1);
}
const clientOptions: ClientOptions = { node: argv.node };
const client = new Client(clientOptions);
if (argv.delete) {
await deleteIndices(
[argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex],
@ -219,6 +230,7 @@ async function main() {
const startTime = new Date().getTime();
await indexHostsAndAlerts(
client,
kbnClient,
seed,
argv.numHosts,
argv.numDocs,
@ -227,6 +239,7 @@ async function main() {
argv.eventIndex,
argv.alertIndex,
argv.alertsPerHost,
argv.fleet,
{
ancestors: argv.ancestors,
generations: argv.generations,