diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts index 248830e02449..c7949299c68d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/constants.ts +++ b/x-pack/plugins/security_solution/common/endpoint/constants.ts @@ -5,6 +5,9 @@ * 2.0. */ +export const ENDPOINT_ACTIONS_INDEX = '.logs-endpoint.actions-default'; +export const ENDPOINT_ACTION_RESPONSES_INDEX = '.logs-endpoint.action.responses-default'; + export const eventsIndexPattern = 'logs-endpoint.events.*'; export const alertsIndexPattern = 'logs-endpoint.alerts-*'; diff --git a/x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts b/x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts new file mode 100644 index 000000000000..0a39e4ea351f --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts @@ -0,0 +1,134 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { DeepPartial } from 'utility-types'; +import { merge } from 'lodash'; +import { BaseDataGenerator } from './base_data_generator'; +import { EndpointActionData, ISOLATION_ACTIONS } from '../types'; + +interface EcsError { + code: string; + id: string; + message: string; + stack_trace: string; + type: string; +} + +interface EndpointActionFields { + action_id: string; + data: EndpointActionData; +} + +interface ActionRequestFields { + expiration: string; + type: 'INPUT_ACTION'; + input_type: 'endpoint'; +} + +interface ActionResponseFields { + completed_at: string; + started_at: string; +} +export interface LogsEndpointAction { + '@timestamp': string; + agent: { + id: string | string[]; + }; + EndpointAction: EndpointActionFields & ActionRequestFields; + error?: EcsError; + user: { + id: string; + }; +} + +export interface LogsEndpointActionResponse { + '@timestamp': string; + agent: { + id: string | string[]; + }; + EndpointAction: EndpointActionFields & ActionResponseFields; + error?: EcsError; +} + +const ISOLATION_COMMANDS: ISOLATION_ACTIONS[] = ['isolate', 'unisolate']; + +export class EndpointActionGenerator extends BaseDataGenerator { + /** Generate a random endpoint Action request (isolate or unisolate) */ + generate(overrides: DeepPartial = {}): LogsEndpointAction { + const timeStamp = new Date(this.randomPastDate()); + return merge( + { + '@timestamp': timeStamp.toISOString(), + agent: { + id: [this.randomUUID()], + }, + EndpointAction: { + action_id: this.randomUUID(), + expiration: this.randomFutureDate(timeStamp), + type: 'INPUT_ACTION', + input_type: 'endpoint', + data: { + command: this.randomIsolateCommand(), + comment: this.randomString(15), + }, + }, + error: undefined, + user: { + id: this.randomUser(), + }, + }, + overrides + ); + } + + generateIsolateAction(overrides: DeepPartial = {}): LogsEndpointAction { + return merge(this.generate({ EndpointAction: { data: { command: 'isolate' } } }), overrides); + } + + generateUnIsolateAction(overrides: DeepPartial = {}): LogsEndpointAction { + return merge(this.generate({ EndpointAction: { data: { command: 'unisolate' } } }), overrides); + } + + /** Generates an endpoint action response */ + generateResponse( + overrides: DeepPartial = {} + ): LogsEndpointActionResponse { + const timeStamp = new Date(); + + return merge( + { + '@timestamp': timeStamp.toISOString(), + agent: { + id: this.randomUUID(), + }, + EndpointAction: { + action_id: this.randomUUID(), + completed_at: timeStamp.toISOString(), + data: { + command: this.randomIsolateCommand(), + comment: '', + }, + started_at: this.randomPastDate(), + }, + error: undefined, + }, + overrides + ); + } + + randomFloat(): number { + return this.random(); + } + + randomN(max: number): number { + return super.randomN(max); + } + + protected randomIsolateCommand() { + return this.randomChoice(ISOLATION_COMMANDS); + } +} diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_actions.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_actions.ts new file mode 100644 index 000000000000..bf46214b20f3 --- /dev/null +++ b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_actions.ts @@ -0,0 +1,217 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { Client } from '@elastic/elasticsearch'; +import { DeleteByQueryResponse } from '@elastic/elasticsearch/api/types'; +import { HostMetadata } from '../types'; +import { + EndpointActionGenerator, + LogsEndpointAction, + LogsEndpointActionResponse, +} from '../data_generators/endpoint_action_generator'; +import { wrapErrorAndRejectPromise } from './utils'; +import { ENDPOINT_ACTIONS_INDEX, ENDPOINT_ACTION_RESPONSES_INDEX } from '../constants'; + +const defaultEndpointActionGenerator = new EndpointActionGenerator(); + +export interface IndexedEndpointActionsForHostResponse { + endpointActions: LogsEndpointAction[]; + endpointActionResponses: LogsEndpointActionResponse[]; + endpointActionsIndex: string; + endpointActionResponsesIndex: string; +} + +/** + * Indexes a random number of Endpoint Actions for a given host + * + * @param esClient + * @param endpointHost + * @param [endpointActionGenerator] + */ +export const indexEndpointActionsForHost = async ( + esClient: Client, + endpointHost: HostMetadata, + endpointActionGenerator: EndpointActionGenerator = defaultEndpointActionGenerator +): Promise => { + const agentId = endpointHost.elastic.agent.id; + const total = endpointActionGenerator.randomN(5); + const response: IndexedEndpointActionsForHostResponse = { + endpointActions: [], + endpointActionResponses: [], + endpointActionsIndex: ENDPOINT_ACTIONS_INDEX, + endpointActionResponsesIndex: ENDPOINT_ACTION_RESPONSES_INDEX, + }; + + for (let i = 0; i < total; i++) { + // create an action + const action = endpointActionGenerator.generate({ + EndpointAction: { + data: { comment: 'data generator: this host is same as bad' }, + }, + }); + + action.agent.id = [agentId]; + + await esClient + .index({ + index: ENDPOINT_ACTIONS_INDEX, + body: action, + }) + .catch(wrapErrorAndRejectPromise); + + // Create an action response for the above + const actionResponse = endpointActionGenerator.generateResponse({ + agent: { id: agentId }, + EndpointAction: { + action_id: action.EndpointAction.action_id, + data: action.EndpointAction.data, + }, + }); + + await esClient + .index({ + index: ENDPOINT_ACTION_RESPONSES_INDEX, + body: actionResponse, + }) + .catch(wrapErrorAndRejectPromise); + + response.endpointActions.push(action); + response.endpointActionResponses.push(actionResponse); + } + + // Add edge cases (maybe) + if (endpointActionGenerator.randomFloat() < 0.3) { + const randomFloat = endpointActionGenerator.randomFloat(); + + // 60% of the time just add either an Isolate -OR- an UnIsolate action + if (randomFloat < 0.6) { + let action: LogsEndpointAction; + + if (randomFloat < 0.3) { + // add a pending isolation + action = endpointActionGenerator.generateIsolateAction({ + '@timestamp': new Date().toISOString(), + }); + } else { + // add a pending UN-isolation + action = endpointActionGenerator.generateUnIsolateAction({ + '@timestamp': new Date().toISOString(), + }); + } + + action.agent.id = [agentId]; + + await esClient + .index({ + index: ENDPOINT_ACTIONS_INDEX, + body: action, + }) + .catch(wrapErrorAndRejectPromise); + + response.endpointActions.push(action); + } else { + // Else (40% of the time) add a pending isolate AND pending un-isolate + const action1 = endpointActionGenerator.generateIsolateAction({ + '@timestamp': new Date().toISOString(), + }); + const action2 = endpointActionGenerator.generateUnIsolateAction({ + '@timestamp': new Date().toISOString(), + }); + + action1.agent.id = [agentId]; + action2.agent.id = [agentId]; + + await Promise.all([ + esClient + .index({ + index: ENDPOINT_ACTIONS_INDEX, + body: action1, + }) + .catch(wrapErrorAndRejectPromise), + esClient + .index({ + index: ENDPOINT_ACTIONS_INDEX, + body: action2, + }) + .catch(wrapErrorAndRejectPromise), + ]); + + response.endpointActions.push(action1, action2); + } + } + + return response; +}; + +export interface DeleteIndexedEndpointActionsResponse { + endpointActionRequests: DeleteByQueryResponse | undefined; + endpointActionResponses: DeleteByQueryResponse | undefined; +} + +export const deleteIndexedEndpointActions = async ( + esClient: Client, + indexedData: IndexedEndpointActionsForHostResponse +): Promise => { + const response: DeleteIndexedEndpointActionsResponse = { + endpointActionRequests: undefined, + endpointActionResponses: undefined, + }; + + if (indexedData.endpointActions.length) { + response.endpointActionRequests = ( + await esClient + .deleteByQuery({ + index: `${indexedData.endpointActionsIndex}-*`, + wait_for_completion: true, + body: { + query: { + bool: { + filter: [ + { + terms: { + action_id: indexedData.endpointActions.map( + (action) => action.EndpointAction.action_id + ), + }, + }, + ], + }, + }, + }, + }) + .catch(wrapErrorAndRejectPromise) + ).body; + } + + if (indexedData.endpointActionResponses) { + response.endpointActionResponses = ( + await esClient + .deleteByQuery({ + index: `${indexedData.endpointActionResponsesIndex}-*`, + wait_for_completion: true, + body: { + query: { + bool: { + filter: [ + { + terms: { + action_id: indexedData.endpointActionResponses.map( + (action) => action.EndpointAction.action_id + ), + }, + }, + ], + }, + }, + }, + }) + .catch(wrapErrorAndRejectPromise) + ).body; + } + + return response; +}; diff --git a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts index 6afd2de5b56b..d7ab014c3b44 100644 --- a/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts +++ b/x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts @@ -26,6 +26,13 @@ import { IndexedFleetActionsForHostResponse, indexFleetActionsForHost, } from './index_fleet_actions'; +import { + deleteIndexedEndpointActions, + DeleteIndexedEndpointActionsResponse, + IndexedEndpointActionsForHostResponse, + indexEndpointActionsForHost, +} from './index_endpoint_actions'; + import { deleteIndexedFleetEndpointPolicies, DeleteIndexedFleetEndpointPoliciesResponse, @@ -38,6 +45,7 @@ import { EndpointDataLoadingError, wrapErrorAndRejectPromise } from './utils'; export interface IndexedHostsResponse extends IndexedFleetAgentResponse, IndexedFleetActionsForHostResponse, + IndexedEndpointActionsForHostResponse, IndexedFleetEndpointPolicyResponse { /** * The documents (1 or more) that were generated for the (single) endpoint host. @@ -81,6 +89,7 @@ export async function indexEndpointHostDocs({ metadataIndex, policyResponseIndex, enrollFleet, + addEndpointActions, generator, }: { numDocs: number; @@ -91,6 +100,7 @@ export async function indexEndpointHostDocs({ metadataIndex: string; policyResponseIndex: string; enrollFleet: boolean; + addEndpointActions: boolean; generator: EndpointDocGenerator; }): Promise { const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents @@ -103,6 +113,10 @@ export async function indexEndpointHostDocs({ metadataIndex, policyResponseIndex, fleetAgentsIndex: '', + endpointActionResponses: [], + endpointActionResponsesIndex: '', + endpointActions: [], + endpointActionsIndex: '', actionResponses: [], responsesIndex: '', actions: [], @@ -177,8 +191,15 @@ export async function indexEndpointHostDocs({ }, }; - // Create some actions for this Host - await indexFleetActionsForHost(client, hostMetadata); + // Create some fleet endpoint actions and .logs-endpoint actions for this Host + if (addEndpointActions) { + await Promise.all([ + indexFleetActionsForHost(client, hostMetadata), + indexEndpointActionsForHost(client, hostMetadata), + ]); + } else { + await indexFleetActionsForHost(client, hostMetadata); + } } hostMetadata = { @@ -237,6 +258,7 @@ const fetchKibanaVersion = async (kbnClient: KbnClient) => { export interface DeleteIndexedEndpointHostsResponse extends DeleteIndexedFleetAgentsResponse, DeleteIndexedFleetActionsResponse, + DeleteIndexedEndpointActionsResponse, DeleteIndexedFleetEndpointPoliciesResponse { hosts: DeleteByQueryResponse | undefined; policyResponses: DeleteByQueryResponse | undefined; @@ -253,6 +275,8 @@ export const deleteIndexedEndpointHosts = async ( agents: undefined, responses: undefined, actions: undefined, + endpointActionRequests: undefined, + endpointActionResponses: undefined, integrationPolicies: undefined, agentPolicies: undefined, }; @@ -314,6 +338,7 @@ export const deleteIndexedEndpointHosts = async ( merge(response, await deleteIndexedFleetAgents(esClient, indexedData)); merge(response, await deleteIndexedFleetActions(esClient, indexedData)); + merge(response, await deleteIndexedEndpointActions(esClient, indexedData)); merge(response, await deleteIndexedFleetEndpointPolicies(kbnClient, indexedData)); return response; diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 2221b2a2d2c9..5bb3bd3dbae5 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -57,6 +57,7 @@ export async function indexHostsAndAlerts( alertIndex: string, alertsPerHost: number, fleet: boolean, + logsEndpoint: boolean, options: TreeOptions = {} ): Promise { const random = seedrandom(seed); @@ -72,11 +73,15 @@ export async function indexHostsAndAlerts( responsesIndex: '', actions: [], actionsIndex: '', + endpointActions: [], + endpointActionsIndex: '', + endpointActionResponses: [], + endpointActionResponsesIndex: '', integrationPolicies: [], agentPolicies: [], }; - // Ensure fleet is setup and endpint package installed + // Ensure fleet is setup and endpoint package installed await setupFleetForEndpoint(kbnClient); // If `fleet` integration is true, then ensure a (fake) fleet-server is connected @@ -98,6 +103,7 @@ export async function indexHostsAndAlerts( metadataIndex, policyResponseIndex, enrollFleet: fleet, + addEndpointActions: logsEndpoint, generator, }); diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts index a9ad30adc994..3c267117964c 100644 --- a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts +++ b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts @@ -165,6 +165,14 @@ async function main() { type: 'boolean', default: false, }, + logsEndpoint: { + alias: 'le', + describe: + 'By default .logs-endpoint.action and .logs-endpoint.action.responses are not indexed. \ + Add endpoint actions and responses using this option. Starting with v7.16.0.', + type: 'boolean', + default: false, + }, ssl: { alias: 'ssl', describe: 'Use https for elasticsearch and kbn clients', @@ -226,6 +234,7 @@ async function main() { argv.alertIndex, argv.alertsPerHost, argv.fleet, + argv.logsEndpoint, { ancestors: argv.ancestors, generations: argv.generations, diff --git a/x-pack/test/security_solution_endpoint/services/endpoint.ts b/x-pack/test/security_solution_endpoint/services/endpoint.ts index a7bc8609b0a0..2e774dcd8478 100644 --- a/x-pack/test/security_solution_endpoint/services/endpoint.ts +++ b/x-pack/test/security_solution_endpoint/services/endpoint.ts @@ -91,6 +91,7 @@ export class EndpointTestResources extends FtrService { numHostDocs: number; alertsPerHost: number; enableFleetIntegration: boolean; + logsEndpoint: boolean; generatorSeed: string; waitUntilTransformed: boolean; }> = {} @@ -100,6 +101,7 @@ export class EndpointTestResources extends FtrService { numHostDocs = 1, alertsPerHost = 1, enableFleetIntegration = true, + logsEndpoint = false, generatorSeed = 'seed', waitUntilTransformed = true, } = options; @@ -116,7 +118,8 @@ export class EndpointTestResources extends FtrService { 'logs-endpoint.events.process-default', 'logs-endpoint.alerts-default', alertsPerHost, - enableFleetIntegration + enableFleetIntegration, + logsEndpoint ); if (waitUntilTransformed) {