[SIEM] [Detection Engine] Search signals index (#52661)

* adds route for querying signals index, also updates signal status type names

* first pass at happy path tests

* fixes stuff after rebase with master

* utilizes removes search_query from payload and replaces it with just query, adds aggs to signals search api, updates route and validation tests

* removes _headers parameter from route handler and updates comment for aggs script

x-pack/legacy/plugins/siem/common/constants.ts

@@ -53,3 +53,4 @@ export const DETECTION_ENGINE_INDEX_URL = `${DETECTION_ENGINE_URL}/index`;
 export const SIGNALS_INDEX_KEY = 'signalsIndex';
 export const DETECTION_ENGINE_SIGNALS_URL = `${DETECTION_ENGINE_URL}/signals`;
 export const DETECTION_ENGINE_SIGNALS_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/status`;
+export const DETECTION_ENGINE_QUERY_SIGNALS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/search`;

x-pack/legacy/plugins/siem/server/kibana.index.ts

@@ -15,6 +15,7 @@ import { findRulesRoute } from './lib/detection_engine/routes/rules/find_rules_r
 import { deleteRulesRoute } from './lib/detection_engine/routes/rules/delete_rules_route';
 import { updateRulesRoute } from './lib/detection_engine/routes/rules/update_rules_route';
 import { setSignalsStatusRoute } from './lib/detection_engine/routes/signals/open_close_signals_route';
+import { querySignalsRoute } from './lib/detection_engine/routes/signals/query_signals_route';
 import { ServerFacade } from './types';
 import { deleteIndexRoute } from './lib/detection_engine/routes/index/delete_index_route';
 import { isAlertExecutor } from './lib/detection_engine/signals/types';
@@ -44,6 +45,7 @@ export const initServerWithKibana = (context: PluginInitializerContext, __legacy
   // POST /api/detection_engine/signals/status
   // Example usage can be found in siem/server/lib/detection_engine/scripts/signals
   setSignalsStatusRoute(__legacy);
+  querySignalsRoute(__legacy);
 
   // Detection Engine index routes that have the REST endpoints of /api/detection_engine/index
   // All REST index creation, policy management for spaces

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -6,10 +6,11 @@
 
 import { ServerInjectOptions } from 'hapi';
 import { ActionResult } from '../../../../../../actions/server/types';
-import { SignalsRestParams } from '../../signals/types';
+import { SignalsStatusRestParams, SignalsQueryRestParams } from '../../signals/types';
 import {
   DETECTION_ENGINE_RULES_URL,
   DETECTION_ENGINE_SIGNALS_STATUS_URL,
+  DETECTION_ENGINE_QUERY_SIGNALS_URL,
 } from '../../../../../common/constants';
 import { RuleAlertType } from '../../rules/types';
 import { RuleAlertParamsRest } from '../../types';
@@ -40,17 +41,25 @@ export const typicalPayload = (): Partial<Omit<RuleAlertParamsRest, 'filter'>> =
   ],
 });
 
-export const typicalSetStatusSignalByIdsPayload = (): Partial<SignalsRestParams> => ({
+export const typicalSetStatusSignalByIdsPayload = (): Partial<SignalsStatusRestParams> => ({
   signal_ids: ['somefakeid1', 'somefakeid2'],
   status: 'closed',
 });
 
-export const typicalSetStatusSignalByQueryPayload = (): Partial<SignalsRestParams> => ({
+export const typicalSetStatusSignalByQueryPayload = (): Partial<SignalsStatusRestParams> => ({
   query: { range: { '@timestamp': { gte: 'now-2M', lte: 'now/M' } } },
   status: 'closed',
 });
 
-export const setStatusSignalMissingIdsAndQueryPayload = (): Partial<SignalsRestParams> => ({
+export const typicalSignalsQuery = (): Partial<SignalsQueryRestParams> => ({
+  query: { match_all: {} },
+});
+
+export const typicalSignalsQueryAggs = (): Partial<SignalsQueryRestParams> => ({
+  aggs: { statuses: { terms: { field: 'signal.status', size: 10 } } },
+});
+
+export const setStatusSignalMissingIdsAndQueryPayload = (): Partial<SignalsStatusRestParams> => ({
   status: 'closed',
 });
 
@@ -134,6 +143,18 @@ export const getSetSignalStatusByQueryRequest = (): ServerInjectOptions => ({
   },
 });
 
+export const getSignalsQueryRequest = (): ServerInjectOptions => ({
+  method: 'POST',
+  url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+  payload: { ...typicalSignalsQuery() },
+});
+
+export const getSignalsAggsQueryRequest = (): ServerInjectOptions => ({
+  method: 'POST',
+  url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+  payload: { ...typicalSignalsQueryAggs() },
+});
+
 export const createActionResult = (): ActionResult => ({
   id: 'result-1',
   actionTypeId: 'action-id-1',

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.test.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { querySignalsSchema } from './query_signals_index_schema';
+import { SignalsQueryRestParams } from '../../signals/types';
+
+describe('query and aggs on signals index', () => {
+  test('query and aggs simultaneously', () => {
+    expect(
+      querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
+        query: {},
+        aggs: {},
+      }).error
+    ).toBeFalsy();
+  });
+
+  test('query only', () => {
+    expect(
+      querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
+        query: {},
+      }).error
+    ).toBeFalsy();
+  });
+
+  test('aggs only', () => {
+    expect(
+      querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({
+        aggs: {},
+      }).error
+    ).toBeFalsy();
+  });
+
+  test('missing query and aggs is invalid', () => {
+    expect(querySignalsSchema.validate<Partial<SignalsQueryRestParams>>({}).error).toBeTruthy();
+  });
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/query_signals_index_schema.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Joi from 'joi';
+
+export const querySignalsSchema = Joi.object({
+  query: Joi.object(),
+  aggs: Joi.object(),
+}).min(1);

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/set_signal_status_schema.test.ts

@@ -5,12 +5,12 @@
  */
 
 import { setSignalsStatusSchema } from './set_signal_status_schema';
-import { SignalsRestParams } from '../../signals/types';
+import { SignalsStatusRestParams } from '../../signals/types';
 
 describe('set signal status schema', () => {
   test('signal_ids and status is valid', () => {
     expect(
-      setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
+      setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
         signal_ids: ['somefakeid'],
         status: 'open',
       }).error
@@ -19,7 +19,7 @@ describe('set signal status schema', () => {
 
   test('query and status is valid', () => {
     expect(
-      setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
+      setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
         query: {},
         status: 'open',
       }).error
@@ -28,7 +28,7 @@ describe('set signal status schema', () => {
 
   test('signal_ids and missing status is invalid', () => {
     expect(
-      setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
+      setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
         signal_ids: ['somefakeid'],
       }).error
     ).toBeTruthy();
@@ -36,7 +36,7 @@ describe('set signal status schema', () => {
 
   test('query and missing status is invalid', () => {
     expect(
-      setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
+      setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
         query: {},
       }).error
     ).toBeTruthy();
@@ -44,7 +44,7 @@ describe('set signal status schema', () => {
 
   test('status is present but query or signal_ids is missing is invalid', () => {
     expect(
-      setSignalsStatusSchema.validate<Partial<SignalsRestParams>>({
+      setSignalsStatusSchema.validate<Partial<SignalsStatusRestParams>>({
         status: 'closed',
       }).error
     ).toBeTruthy();
@@ -54,7 +54,7 @@ describe('set signal status schema', () => {
     expect(
       setSignalsStatusSchema.validate<
         Partial<
-          Omit<SignalsRestParams, 'status'> & {
+          Omit<SignalsStatusRestParams, 'status'> & {
             status: string;
           }
         >

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals_route.ts

@@ -6,7 +6,7 @@
 
 import Hapi from 'hapi';
 import { DETECTION_ENGINE_SIGNALS_STATUS_URL } from '../../../../../common/constants';
-import { SignalsRequest } from '../../signals/types';
+import { SignalsStatusRequest } from '../../signals/types';
 import { setSignalsStatusSchema } from '../schemas/set_signal_status_schema';
 import { ServerFacade } from '../../../../types';
 import { transformError, getIndex } from '../utils';
@@ -24,7 +24,7 @@ export const setSignalsStatusRouteDef = (server: ServerFacade): Hapi.ServerRoute
         payload: setSignalsStatusSchema,
       },
     },
-    async handler(request: SignalsRequest, headers) {
+    async handler(request: SignalsStatusRequest) {
       const { signal_ids: signalIds, query, status } = request.payload;
       const index = getIndex(request, server);
       const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { createMockServer } from '../__mocks__/_mock_server';
+import { querySignalsRoute } from './query_signals_route';
+import * as myUtils from '../utils';
+import { ServerInjectOptions } from 'hapi';
+import {
+  getSignalsQueryRequest,
+  getSignalsAggsQueryRequest,
+  typicalSignalsQuery,
+  typicalSignalsQueryAggs,
+} from '../__mocks__/request_responses';
+import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants';
+
+describe('query for signal', () => {
+  let { server, elasticsearch } = createMockServer();
+
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.spyOn(myUtils, 'getIndex').mockReturnValue('fakeindex');
+    ({ server, elasticsearch } = createMockServer());
+    elasticsearch.getCluster = jest.fn(() => ({
+      callWithRequest: jest.fn(() => true),
+    }));
+    querySignalsRoute(server);
+  });
+
+  describe('query and agg on signals index', () => {
+    test('returns 200 when using single query', async () => {
+      elasticsearch.getCluster = jest.fn(() => ({
+        callWithRequest: jest.fn(
+          (_req, _type: string, queryBody: { index: string; body: object }) => {
+            expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() });
+            return true;
+          }
+        ),
+      }));
+      const { statusCode } = await server.inject(getSignalsAggsQueryRequest());
+      expect(statusCode).toBe(200);
+      expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
+    });
+
+    test('returns 200 when using single agg', async () => {
+      elasticsearch.getCluster = jest.fn(() => ({
+        callWithRequest: jest.fn(
+          (_req, _type: string, queryBody: { index: string; body: object }) => {
+            expect(queryBody.body).toMatchObject({ ...typicalSignalsQueryAggs() });
+            return true;
+          }
+        ),
+      }));
+      const { statusCode } = await server.inject(getSignalsAggsQueryRequest());
+      expect(statusCode).toBe(200);
+      expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
+    });
+
+    test('returns 200 when using aggs and query together', async () => {
+      const allTogether = getSignalsQueryRequest();
+      allTogether.payload = { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() };
+      elasticsearch.getCluster = jest.fn(() => ({
+        callWithRequest: jest.fn(
+          (_req, _type: string, queryBody: { index: string; body: object }) => {
+            expect(queryBody.body).toMatchObject({
+              ...typicalSignalsQueryAggs(),
+              ...typicalSignalsQuery(),
+            });
+            return true;
+          }
+        ),
+      }));
+      const { statusCode } = await server.inject(allTogether);
+      expect(statusCode).toBe(200);
+      expect(myUtils.getIndex).toHaveReturnedWith('fakeindex');
+    });
+
+    test('returns 400 when missing aggs and query', async () => {
+      const allTogether = getSignalsQueryRequest();
+      allTogether.payload = {};
+      const { statusCode } = await server.inject(allTogether);
+      expect(statusCode).toBe(400);
+    });
+  });
+
+  describe('validation', () => {
+    test('returns 200 if query present', async () => {
+      const request: ServerInjectOptions = {
+        method: 'POST',
+        url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+        payload: typicalSignalsQuery(),
+      };
+      const { statusCode } = await server.inject(request);
+      expect(statusCode).toBe(200);
+    });
+
+    test('returns 200 if aggs is present', async () => {
+      const request: ServerInjectOptions = {
+        method: 'POST',
+        url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+        payload: typicalSignalsQueryAggs(),
+      };
+      const { statusCode } = await server.inject(request);
+      expect(statusCode).toBe(200);
+    });
+
+    test('returns 200 if aggs and query are present', async () => {
+      const request: ServerInjectOptions = {
+        method: 'POST',
+        url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+        payload: { ...typicalSignalsQueryAggs(), ...typicalSignalsQuery() },
+      };
+      const { statusCode } = await server.inject(request);
+      expect(statusCode).toBe(200);
+    });
+
+    test('returns 400 if aggs and query are NOT present', async () => {
+      const request: ServerInjectOptions = {
+        method: 'POST',
+        url: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+        payload: {},
+      };
+      const { statusCode } = await server.inject(request);
+      expect(statusCode).toBe(400);
+    });
+  });
+});

x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.ts

@@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Hapi from 'hapi';
+import { DETECTION_ENGINE_QUERY_SIGNALS_URL } from '../../../../../common/constants';
+import { SignalsQueryRequest } from '../../signals/types';
+import { querySignalsSchema } from '../schemas/query_signals_index_schema';
+import { ServerFacade } from '../../../../types';
+import { transformError, getIndex } from '../utils';
+
+export const querySignalsRouteDef = (server: ServerFacade): Hapi.ServerRoute => {
+  return {
+    method: 'POST',
+    path: DETECTION_ENGINE_QUERY_SIGNALS_URL,
+    options: {
+      tags: ['access:siem'],
+      validate: {
+        options: {
+          abortEarly: false,
+        },
+        payload: querySignalsSchema,
+      },
+    },
+    async handler(request: SignalsQueryRequest) {
+      const { query, aggs } = request.payload;
+      const body = { query, aggs };
+      const index = getIndex(request, server);
+      const { callWithRequest } = server.plugins.elasticsearch.getCluster('data');
+      try {
+        return callWithRequest(request, 'search', {
+          index,
+          body,
+        });
+      } catch (exc) {
+        // error while getting or updating signal with id: id in signal index .siem-signals
+        return transformError(exc);
+      }
+    },
+  };
+};
+
+export const querySignalsRoute = (server: ServerFacade) => {
+  server.route(querySignalsRouteDef(server));
+};

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/aggs_signals.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./signals/aggs_signal.sh
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/search \
+  -d '{"aggs": {"statuses": {"terms": {"field": "signal.status", "size": 10 }}}}' \
+  | jq .

x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/query_signals.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License;
+# you may not use this file except in compliance with the Elastic License.
+#
+
+set -e
+./check_env_variables.sh
+
+# Example: ./signals/query_signals.sh
+  curl -s -k \
+  -H 'Content-Type: application/json' \
+  -H 'kbn-xsrf: 123' \
+  -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+  -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/search \
+  -d '{ "query": { "match_all": {} } } ' \
+  | jq .

x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/types.ts

@@ -15,12 +15,29 @@ export interface SignalsParams {
   status: 'open' | 'closed';
 }
 
-export type SignalsRestParams = Omit<SignalsParams, 'signalIds'> & {
-  signal_ids: SignalsParams['signalIds'];
+export interface SignalsStatusParams {
+  signalIds: string[] | undefined | null;
+  query: object | undefined | null;
+  status: 'open' | 'closed';
+}
+
+export interface SignalQueryParams {
+  query: object | undefined | null;
+  aggs: object | undefined | null;
+}
+
+export type SignalsStatusRestParams = Omit<SignalsStatusParams, 'signalIds'> & {
+  signal_ids: SignalsStatusParams['signalIds'];
 };
 
-export interface SignalsRequest extends RequestFacade {
-  payload: SignalsRestParams;
+export type SignalsQueryRestParams = SignalQueryParams;
+
+export interface SignalsStatusRequest extends RequestFacade {
+  payload: SignalsStatusRestParams;
+}
+
+export interface SignalsQueryRequest extends RequestFacade {
+  payload: SignalsQueryRestParams;
 }
 
 export type SearchTypes =