[Security Solution][Endpoint] Action and responses data generators for .logs-endpoint.action* data streams (#113403) (#114112)

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -5,6 +5,9 @@
  * 2.0.
  */
 
+export const ENDPOINT_ACTIONS_INDEX = '.logs-endpoint.actions-default';
+export const ENDPOINT_ACTION_RESPONSES_INDEX = '.logs-endpoint.action.responses-default';
+
 export const eventsIndexPattern = 'logs-endpoint.events.*';
 export const alertsIndexPattern = 'logs-endpoint.alerts-*';
 

x-pack/plugins/security_solution/common/endpoint/data_generators/endpoint_action_generator.ts

@@ -0,0 +1,134 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { DeepPartial } from 'utility-types';
+import { merge } from 'lodash';
+import { BaseDataGenerator } from './base_data_generator';
+import { EndpointActionData, ISOLATION_ACTIONS } from '../types';
+
+interface EcsError {
+  code: string;
+  id: string;
+  message: string;
+  stack_trace: string;
+  type: string;
+}
+
+interface EndpointActionFields {
+  action_id: string;
+  data: EndpointActionData;
+}
+
+interface ActionRequestFields {
+  expiration: string;
+  type: 'INPUT_ACTION';
+  input_type: 'endpoint';
+}
+
+interface ActionResponseFields {
+  completed_at: string;
+  started_at: string;
+}
+export interface LogsEndpointAction {
+  '@timestamp': string;
+  agent: {
+    id: string | string[];
+  };
+  EndpointAction: EndpointActionFields & ActionRequestFields;
+  error?: EcsError;
+  user: {
+    id: string;
+  };
+}
+
+export interface LogsEndpointActionResponse {
+  '@timestamp': string;
+  agent: {
+    id: string | string[];
+  };
+  EndpointAction: EndpointActionFields & ActionResponseFields;
+  error?: EcsError;
+}
+
+const ISOLATION_COMMANDS: ISOLATION_ACTIONS[] = ['isolate', 'unisolate'];
+
+export class EndpointActionGenerator extends BaseDataGenerator {
+  /** Generate a random endpoint Action request (isolate or unisolate) */
+  generate(overrides: DeepPartial<LogsEndpointAction> = {}): LogsEndpointAction {
+    const timeStamp = new Date(this.randomPastDate());
+    return merge(
+      {
+        '@timestamp': timeStamp.toISOString(),
+        agent: {
+          id: [this.randomUUID()],
+        },
+        EndpointAction: {
+          action_id: this.randomUUID(),
+          expiration: this.randomFutureDate(timeStamp),
+          type: 'INPUT_ACTION',
+          input_type: 'endpoint',
+          data: {
+            command: this.randomIsolateCommand(),
+            comment: this.randomString(15),
+          },
+        },
+        error: undefined,
+        user: {
+          id: this.randomUser(),
+        },
+      },
+      overrides
+    );
+  }
+
+  generateIsolateAction(overrides: DeepPartial<LogsEndpointAction> = {}): LogsEndpointAction {
+    return merge(this.generate({ EndpointAction: { data: { command: 'isolate' } } }), overrides);
+  }
+
+  generateUnIsolateAction(overrides: DeepPartial<LogsEndpointAction> = {}): LogsEndpointAction {
+    return merge(this.generate({ EndpointAction: { data: { command: 'unisolate' } } }), overrides);
+  }
+
+  /** Generates an endpoint action response */
+  generateResponse(
+    overrides: DeepPartial<LogsEndpointActionResponse> = {}
+  ): LogsEndpointActionResponse {
+    const timeStamp = new Date();
+
+    return merge(
+      {
+        '@timestamp': timeStamp.toISOString(),
+        agent: {
+          id: this.randomUUID(),
+        },
+        EndpointAction: {
+          action_id: this.randomUUID(),
+          completed_at: timeStamp.toISOString(),
+          data: {
+            command: this.randomIsolateCommand(),
+            comment: '',
+          },
+          started_at: this.randomPastDate(),
+        },
+        error: undefined,
+      },
+      overrides
+    );
+  }
+
+  randomFloat(): number {
+    return this.random();
+  }
+
+  randomN(max: number): number {
+    return super.randomN(max);
+  }
+
+  protected randomIsolateCommand() {
+    return this.randomChoice(ISOLATION_COMMANDS);
+  }
+}

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_actions.ts

@@ -0,0 +1,217 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { DeleteByQueryResponse } from '@elastic/elasticsearch/api/types';
+import { HostMetadata } from '../types';
+import {
+  EndpointActionGenerator,
+  LogsEndpointAction,
+  LogsEndpointActionResponse,
+} from '../data_generators/endpoint_action_generator';
+import { wrapErrorAndRejectPromise } from './utils';
+import { ENDPOINT_ACTIONS_INDEX, ENDPOINT_ACTION_RESPONSES_INDEX } from '../constants';
+
+const defaultEndpointActionGenerator = new EndpointActionGenerator();
+
+export interface IndexedEndpointActionsForHostResponse {
+  endpointActions: LogsEndpointAction[];
+  endpointActionResponses: LogsEndpointActionResponse[];
+  endpointActionsIndex: string;
+  endpointActionResponsesIndex: string;
+}
+
+/**
+ * Indexes a random number of Endpoint Actions for a given host
+ *
+ * @param esClient
+ * @param endpointHost
+ * @param [endpointActionGenerator]
+ */
+export const indexEndpointActionsForHost = async (
+  esClient: Client,
+  endpointHost: HostMetadata,
+  endpointActionGenerator: EndpointActionGenerator = defaultEndpointActionGenerator
+): Promise<IndexedEndpointActionsForHostResponse> => {
+  const agentId = endpointHost.elastic.agent.id;
+  const total = endpointActionGenerator.randomN(5);
+  const response: IndexedEndpointActionsForHostResponse = {
+    endpointActions: [],
+    endpointActionResponses: [],
+    endpointActionsIndex: ENDPOINT_ACTIONS_INDEX,
+    endpointActionResponsesIndex: ENDPOINT_ACTION_RESPONSES_INDEX,
+  };
+
+  for (let i = 0; i < total; i++) {
+    // create an action
+    const action = endpointActionGenerator.generate({
+      EndpointAction: {
+        data: { comment: 'data generator: this host is same as bad' },
+      },
+    });
+
+    action.agent.id = [agentId];
+
+    await esClient
+      .index({
+        index: ENDPOINT_ACTIONS_INDEX,
+        body: action,
+      })
+      .catch(wrapErrorAndRejectPromise);
+
+    // Create an action response for the above
+    const actionResponse = endpointActionGenerator.generateResponse({
+      agent: { id: agentId },
+      EndpointAction: {
+        action_id: action.EndpointAction.action_id,
+        data: action.EndpointAction.data,
+      },
+    });
+
+    await esClient
+      .index({
+        index: ENDPOINT_ACTION_RESPONSES_INDEX,
+        body: actionResponse,
+      })
+      .catch(wrapErrorAndRejectPromise);
+
+    response.endpointActions.push(action);
+    response.endpointActionResponses.push(actionResponse);
+  }
+
+  // Add edge cases (maybe)
+  if (endpointActionGenerator.randomFloat() < 0.3) {
+    const randomFloat = endpointActionGenerator.randomFloat();
+
+    // 60% of the time just add either an Isolate -OR- an UnIsolate action
+    if (randomFloat < 0.6) {
+      let action: LogsEndpointAction;
+
+      if (randomFloat < 0.3) {
+        // add a pending isolation
+        action = endpointActionGenerator.generateIsolateAction({
+          '@timestamp': new Date().toISOString(),
+        });
+      } else {
+        // add a pending UN-isolation
+        action = endpointActionGenerator.generateUnIsolateAction({
+          '@timestamp': new Date().toISOString(),
+        });
+      }
+
+      action.agent.id = [agentId];
+
+      await esClient
+        .index({
+          index: ENDPOINT_ACTIONS_INDEX,
+          body: action,
+        })
+        .catch(wrapErrorAndRejectPromise);
+
+      response.endpointActions.push(action);
+    } else {
+      // Else (40% of the time) add a pending isolate AND pending un-isolate
+      const action1 = endpointActionGenerator.generateIsolateAction({
+        '@timestamp': new Date().toISOString(),
+      });
+      const action2 = endpointActionGenerator.generateUnIsolateAction({
+        '@timestamp': new Date().toISOString(),
+      });
+
+      action1.agent.id = [agentId];
+      action2.agent.id = [agentId];
+
+      await Promise.all([
+        esClient
+          .index({
+            index: ENDPOINT_ACTIONS_INDEX,
+            body: action1,
+          })
+          .catch(wrapErrorAndRejectPromise),
+        esClient
+          .index({
+            index: ENDPOINT_ACTIONS_INDEX,
+            body: action2,
+          })
+          .catch(wrapErrorAndRejectPromise),
+      ]);
+
+      response.endpointActions.push(action1, action2);
+    }
+  }
+
+  return response;
+};
+
+export interface DeleteIndexedEndpointActionsResponse {
+  endpointActionRequests: DeleteByQueryResponse | undefined;
+  endpointActionResponses: DeleteByQueryResponse | undefined;
+}
+
+export const deleteIndexedEndpointActions = async (
+  esClient: Client,
+  indexedData: IndexedEndpointActionsForHostResponse
+): Promise<DeleteIndexedEndpointActionsResponse> => {
+  const response: DeleteIndexedEndpointActionsResponse = {
+    endpointActionRequests: undefined,
+    endpointActionResponses: undefined,
+  };
+
+  if (indexedData.endpointActions.length) {
+    response.endpointActionRequests = (
+      await esClient
+        .deleteByQuery({
+          index: `${indexedData.endpointActionsIndex}-*`,
+          wait_for_completion: true,
+          body: {
+            query: {
+              bool: {
+                filter: [
+                  {
+                    terms: {
+                      action_id: indexedData.endpointActions.map(
+                        (action) => action.EndpointAction.action_id
+                      ),
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        })
+        .catch(wrapErrorAndRejectPromise)
+    ).body;
+  }
+
+  if (indexedData.endpointActionResponses) {
+    response.endpointActionResponses = (
+      await esClient
+        .deleteByQuery({
+          index: `${indexedData.endpointActionResponsesIndex}-*`,
+          wait_for_completion: true,
+          body: {
+            query: {
+              bool: {
+                filter: [
+                  {
+                    terms: {
+                      action_id: indexedData.endpointActionResponses.map(
+                        (action) => action.EndpointAction.action_id
+                      ),
+                    },
+                  },
+                ],
+              },
+            },
+          },
+        })
+        .catch(wrapErrorAndRejectPromise)
+    ).body;
+  }
+
+  return response;
+};

x-pack/plugins/security_solution/common/endpoint/data_loaders/index_endpoint_hosts.ts

@@ -26,6 +26,13 @@ import {
   IndexedFleetActionsForHostResponse,
   indexFleetActionsForHost,
 } from './index_fleet_actions';
+import {
+  deleteIndexedEndpointActions,
+  DeleteIndexedEndpointActionsResponse,
+  IndexedEndpointActionsForHostResponse,
+  indexEndpointActionsForHost,
+} from './index_endpoint_actions';
+
 import {
   deleteIndexedFleetEndpointPolicies,
   DeleteIndexedFleetEndpointPoliciesResponse,
@@ -38,6 +45,7 @@ import { EndpointDataLoadingError, wrapErrorAndRejectPromise } from './utils';
 export interface IndexedHostsResponse
   extends IndexedFleetAgentResponse,
     IndexedFleetActionsForHostResponse,
+    IndexedEndpointActionsForHostResponse,
     IndexedFleetEndpointPolicyResponse {
   /**
    * The documents (1 or more) that were generated for the (single) endpoint host.
@@ -81,6 +89,7 @@ export async function indexEndpointHostDocs({
   metadataIndex,
   policyResponseIndex,
   enrollFleet,
+  addEndpointActions,
   generator,
 }: {
   numDocs: number;
@@ -91,6 +100,7 @@ export async function indexEndpointHostDocs({
   metadataIndex: string;
   policyResponseIndex: string;
   enrollFleet: boolean;
+  addEndpointActions: boolean;
   generator: EndpointDocGenerator;
 }): Promise<IndexedHostsResponse> {
   const timeBetweenDocs = 6 * 3600 * 1000; // 6 hours between metadata documents
@@ -103,6 +113,10 @@ export async function indexEndpointHostDocs({
     metadataIndex,
     policyResponseIndex,
     fleetAgentsIndex: '',
+    endpointActionResponses: [],
+    endpointActionResponsesIndex: '',
+    endpointActions: [],
+    endpointActionsIndex: '',
     actionResponses: [],
     responsesIndex: '',
     actions: [],
@@ -177,8 +191,15 @@ export async function indexEndpointHostDocs({
         },
       };
 
-      // Create some actions for this Host
-      await indexFleetActionsForHost(client, hostMetadata);
+      // Create some fleet endpoint actions and .logs-endpoint actions for this Host
+      if (addEndpointActions) {
+        await Promise.all([
+          indexFleetActionsForHost(client, hostMetadata),
+          indexEndpointActionsForHost(client, hostMetadata),
+        ]);
+      } else {
+        await indexFleetActionsForHost(client, hostMetadata);
+      }
     }
 
     hostMetadata = {
@@ -237,6 +258,7 @@ const fetchKibanaVersion = async (kbnClient: KbnClient) => {
 export interface DeleteIndexedEndpointHostsResponse
   extends DeleteIndexedFleetAgentsResponse,
     DeleteIndexedFleetActionsResponse,
+    DeleteIndexedEndpointActionsResponse,
     DeleteIndexedFleetEndpointPoliciesResponse {
   hosts: DeleteByQueryResponse | undefined;
   policyResponses: DeleteByQueryResponse | undefined;
@@ -253,6 +275,8 @@ export const deleteIndexedEndpointHosts = async (
     agents: undefined,
     responses: undefined,
     actions: undefined,
+    endpointActionRequests: undefined,
+    endpointActionResponses: undefined,
     integrationPolicies: undefined,
     agentPolicies: undefined,
   };
@@ -314,6 +338,7 @@ export const deleteIndexedEndpointHosts = async (
 
   merge(response, await deleteIndexedFleetAgents(esClient, indexedData));
   merge(response, await deleteIndexedFleetActions(esClient, indexedData));
+  merge(response, await deleteIndexedEndpointActions(esClient, indexedData));
   merge(response, await deleteIndexedFleetEndpointPolicies(kbnClient, indexedData));
 
   return response;

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -57,6 +57,7 @@ export async function indexHostsAndAlerts(
   alertIndex: string,
   alertsPerHost: number,
   fleet: boolean,
+  logsEndpoint: boolean,
   options: TreeOptions = {}
 ): Promise<IndexedHostsAndAlertsResponse> {
   const random = seedrandom(seed);
@@ -72,11 +73,15 @@ export async function indexHostsAndAlerts(
     responsesIndex: '',
     actions: [],
     actionsIndex: '',
+    endpointActions: [],
+    endpointActionsIndex: '',
+    endpointActionResponses: [],
+    endpointActionResponsesIndex: '',
     integrationPolicies: [],
     agentPolicies: [],
   };
 
-  // Ensure fleet is setup and endpint package installed
+  // Ensure fleet is setup and endpoint package installed
   await setupFleetForEndpoint(kbnClient);
 
   // If `fleet` integration is true, then ensure a (fake) fleet-server is connected
@@ -98,6 +103,7 @@ export async function indexHostsAndAlerts(
       metadataIndex,
       policyResponseIndex,
       enrollFleet: fleet,
+      addEndpointActions: logsEndpoint,
       generator,
     });
 

x-pack/plugins/security_solution/scripts/endpoint/resolver_generator_script.ts

@@ -165,6 +165,14 @@ async function main() {
       type: 'boolean',
       default: false,
     },
+    logsEndpoint: {
+      alias: 'le',
+      describe:
+        'By default .logs-endpoint.action and .logs-endpoint.action.responses are not indexed. \
+        Add endpoint actions and responses using this option. Starting with v7.16.0.',
+      type: 'boolean',
+      default: false,
+    },
     ssl: {
       alias: 'ssl',
       describe: 'Use https for elasticsearch and kbn clients',
@@ -226,6 +234,7 @@ async function main() {
     argv.alertIndex,
     argv.alertsPerHost,
     argv.fleet,
+    argv.logsEndpoint,
     {
       ancestors: argv.ancestors,
       generations: argv.generations,

x-pack/test/security_solution_endpoint/services/endpoint.ts

@@ -91,6 +91,7 @@ export class EndpointTestResources extends FtrService {
       numHostDocs: number;
       alertsPerHost: number;
       enableFleetIntegration: boolean;
+      logsEndpoint: boolean;
       generatorSeed: string;
       waitUntilTransformed: boolean;
     }> = {}
@@ -100,6 +101,7 @@ export class EndpointTestResources extends FtrService {
       numHostDocs = 1,
       alertsPerHost = 1,
       enableFleetIntegration = true,
+      logsEndpoint = false,
       generatorSeed = 'seed',
       waitUntilTransformed = true,
     } = options;
@@ -116,7 +118,8 @@ export class EndpointTestResources extends FtrService {
       'logs-endpoint.events.process-default',
       'logs-endpoint.alerts-default',
       alertsPerHost,
-      enableFleetIntegration
+      enableFleetIntegration,
+      logsEndpoint
     );
 
     if (waitUntilTransformed) {