maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SIEM][Detection Engine] Rule Status Monitoring (#54452) (#54716)

Browse source 
* Working status updates in executor. Need to update read rules api endpoint to only respond with 'status' and not status info. Will create another endpoint to get status details for a rule which will include last five errors (if there are any). Still need tests

* adds new route for getting statuses for a list of given alert ids, adds try-catch and more logic in executor for logging errors, adds scripts and rules for testing, updates find_rules endpoint to display statuses too. Would like to look into using the alerts executor state to better manage logic for statuses, and need to update some types. Also needs unit tests still.

* updated types for routes, updated how merging of alert-to-rule and rule status happens when formatting REST response.

* typecast test server as ServerFacade type

* fix bug where we were not awaiting the accumulated result in the reducer

* update rule status saved object interfaces to play nicely with interfaces provided by saved objects module. Update tests to pass - Need to write new unit tests in an upcoming commit. Next commit will be cleanup from comments then new unit tests.

* fix missed conflicts after rebase

* replace id param with rule.id when searching in statuses, adds sort fields to the saved objects find queries.

* fixes bug where 'executing' statuses were being written into failing historical status list

* camelCase to snake_case in new statuses route, also fix merge conflict

* add deletion of rule statuses to delete_rules_bulk_route. Statuses are created inside of executor so we will not be needing to create statuses directly inside of the create rules bulk route, so I removed that extraneous code.

* pr feedback I forgot to fix earlier

* remove unused import. fixes type check error generated in previous commit

* removes status information from rule when saved to signals index and updates tests to represent this change. Also removes extraneous quotes inserted around alertId field when creating a new historical status.

* adds new bash script to delete all rule statuses, updates error messages in rule statuses to just store actual message, moved querying of rules statuses under a null check, initialize everything to null when first creating rule status, update number of results returned when querying saved objects based on usage, updates saved objects mapping types to use date for dates and keyword for alertId.

* use lodash snake case and update total number of saved objects to return for find rules, delete rules, and read rules.

* updates how statuses are transformed inside of read_rules_route, only update updated_at in rule on update of rule, removes unlabeled todo comment, updates scripts descriptions, removes interval from query_with_rule_id.json sample query, removes debug statement, removes verbose from curl script.

* display rule status on update
This commit is contained in:
Devin W. Hurley 2020-01-14 09:17:11 -05:00 committed by GitHub
parent 2f87d85519
commit 68ff6f6db4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
36 changed files with 758 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
x-pack/legacy/plugins/siem/index.ts
View file

@ -147,6 +147,7 @@ export const siem = (kibana: any) => {
alerting: plugins.alerting,
elasticsearch: plugins.elasticsearch,
spaces: plugins.spaces,
savedObjects: server.savedObjects.SavedObjectsClient,
},
route: route.bind(server),
};

2
x-pack/legacy/plugins/siem/server/kibana.index.ts
View file

@ -27,6 +27,7 @@ import { updateRulesBulkRoute } from './lib/detection_engine/routes/rules/update
import { deleteRulesBulkRoute } from './lib/detection_engine/routes/rules/delete_rules_bulk_route';
import { importRulesRoute } from './lib/detection_engine/routes/rules/import_rules_route';
import { exportRulesRoute } from './lib/detection_engine/routes/rules/export_rules_route';
import { findRulesStatusesRoute } from './lib/detection_engine/routes/rules/find_rules_status_route';
const APP_ID = 'siem';
@ -54,6 +55,7 @@ export const initServerWithKibana = (context: PluginInitializerContext, __legacy
deleteRulesBulkRoute(__legacy);
importRulesRoute(__legacy);
exportRulesRoute(__legacy);
findRulesStatusesRoute(__legacy);
// Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals
// POST /api/detection_engine/signals/status

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/_mock_server.ts
View file

@ -7,7 +7,7 @@
import Hapi from 'hapi';
import { KibanaConfig } from 'src/legacy/server/kbn_server';
import { ElasticsearchPlugin } from 'src/legacy/core_plugins/elasticsearch';
import { savedObjectsClientMock } from '../../../../../../../../../src/core/server/mocks';
import { alertsClientMock } from '../../../../../../alerting/server/alerts_client.mock';
import { actionsClientMock } from '../../../../../../actions/server/actions_client.mock';
@ -48,6 +48,7 @@ export const createMockServer = (config: Record<string, string> = defaultConfig)
const actionsClient = actionsClientMock.create();
const alertsClient = alertsClientMock.create();
const savedObjectsClient = savedObjectsClientMock.create();
const elasticsearch = {
getCluster: jest.fn().mockImplementation(() => ({
callWithRequest: jest.fn(),
@ -57,8 +58,15 @@ export const createMockServer = (config: Record<string, string> = defaultConfig)
server.decorate('request', 'getBasePath', () => '/s/default');
server.decorate('request', 'getActionsClient', () => actionsClient);
server.plugins.elasticsearch = (elasticsearch as unknown) as ElasticsearchPlugin;
server.decorate('request', 'getSavedObjectsClient', () => savedObjectsClient);
return { server, alertsClient, actionsClient, elasticsearch };
return {
server,
alertsClient,
actionsClient,
elasticsearch,
savedObjectsClient,
};
};
export const createMockServerWithoutAlertClientDecoration = (

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
View file

@ -5,6 +5,7 @@
*/
import { ServerInjectOptions } from 'hapi';
import { SavedObjectsFindResponse } from 'kibana/server';
import { ActionResult } from '../../../../../../actions/server/types';
import { SignalsStatusRestParams, SignalsQueryRestParams } from '../../signals/types';
import {
@ -15,7 +16,7 @@ import {
INTERNAL_RULE_ID_KEY,
INTERNAL_IMMUTABLE_KEY,
} from '../../../../../common/constants';
import { RuleAlertType } from '../../rules/types';
import { RuleAlertType, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { RuleAlertParamsRest } from '../../types';
export const fullRuleAlertParamsRest = (): RuleAlertParamsRest => ({
@ -383,3 +384,10 @@ export const getMockPrivileges = () => ({
application: {},
isAuthenticated: false,
});
export const getFindResultStatus = (): SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes> => ({
page: 1,
per_page: 1,
total: 0,
saved_objects: [],
});

3
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/privileges/read_privileges_route.test.ts
View file

@ -7,6 +7,7 @@
import { createMockServer } from '../__mocks__/_mock_server';
import { getPrivilegeRequest, getMockPrivileges } from '../__mocks__/request_responses';
import { readPrivilegesRoute } from './read_privileges_route';
import { ServerFacade } from '../../../../types';
import * as myUtils from '../utils';
describe('read_privileges', () => {
@ -18,7 +19,7 @@ describe('read_privileges', () => {
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(() => getMockPrivileges()),
}));
readPrivilegesRoute(server);
readPrivilegesRoute((server as unknown) as ServerFacade);
});
afterEach(() => {

9
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.test.ts
View file

@ -12,6 +12,7 @@ import {
} from '../__mocks__/_mock_server';
import { createRulesRoute } from './create_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getResult,
@ -32,7 +33,7 @@ describe('create_rules_bulk', () => {
callWithRequest: jest.fn().mockImplementation(() => true),
}));
createRulesBulkRoute(server);
createRulesBulkRoute((server as unknown) as ServerFacade);
});
describe('status codes with actionClient and alertClient', () => {
@ -47,14 +48,14 @@ describe('create_rules_bulk', () => {
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
createRulesRoute(serverWithoutActionClient);
createRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getReadBulkRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
createRulesRoute(serverWithoutAlertClient);
createRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getReadBulkRequest());
expect(statusCode).toBe(404);
});
@ -63,7 +64,7 @@ describe('create_rules_bulk', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
createRulesRoute(serverWithoutActionOrAlertClient);
createRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getReadBulkRequest());
expect(statusCode).toBe(404);
});

6
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_bulk_route.ts
View file

@ -40,8 +40,10 @@ export const createCreateRulesBulkRoute = (server: ServerFacade): Hapi.ServerRou
const actionsClient = isFunction(request.getActionsClient)
? request.getActionsClient()
: null;
if (!alertsClient || !actionsClient) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}

31
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.test.ts
View file

@ -12,28 +12,43 @@ import {
} from '../__mocks__/_mock_server';
import { createRulesRoute } from './create_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getResult,
createActionResult,
getCreateRequest,
typicalPayload,
getFindResultStatus,
} from '../__mocks__/request_responses';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
describe('create_rules', () => {
let { server, alertsClient, actionsClient, elasticsearch } = createMockServer();
let {
server,
alertsClient,
actionsClient,
elasticsearch,
savedObjectsClient,
} = createMockServer();
beforeEach(() => {
jest.resetAllMocks();
({ server, alertsClient, actionsClient, elasticsearch } = createMockServer());
({
server,
alertsClient,
actionsClient,
elasticsearch,
savedObjectsClient,
} = createMockServer());
elasticsearch.getCluster = jest.fn().mockImplementation(() => ({
callWithRequest: jest
.fn()
.mockImplementation((endpoint, params) => ({ _shards: { total: 1 } })),
}));
createRulesRoute(server);
createRulesRoute((server as unknown) as ServerFacade);
});
describe('status codes with actionClient and alertClient', () => {
@ -42,20 +57,21 @@ describe('create_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.create.mockResolvedValue(createActionResult());
alertsClient.create.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { statusCode } = await server.inject(getCreateRequest());
expect(statusCode).toBe(200);
});
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
createRulesRoute(serverWithoutActionClient);
createRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getCreateRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
createRulesRoute(serverWithoutAlertClient);
createRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getCreateRequest());
expect(statusCode).toBe(404);
});
@ -64,7 +80,7 @@ describe('create_rules', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
createRulesRoute(serverWithoutActionOrAlertClient);
createRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getCreateRequest());
expect(statusCode).toBe(404);
});
@ -76,6 +92,7 @@ describe('create_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.create.mockResolvedValue(createActionResult());
alertsClient.create.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
// missing rule_id should return 200 as it will be auto generated if not given
const { rule_id, ...noRuleId } = typicalPayload();
const request: ServerInjectOptions = {
@ -92,6 +109,7 @@ describe('create_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.create.mockResolvedValue(createActionResult());
alertsClient.create.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { type, ...noType } = typicalPayload();
const request: ServerInjectOptions = {
method: 'POST',
@ -110,6 +128,7 @@ describe('create_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.create.mockResolvedValue(createActionResult());
alertsClient.create.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { type, ...noType } = typicalPayload();
const request: ServerInjectOptions = {
method: 'POST',

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/create_rules_route.ts
View file

@ -10,10 +10,11 @@ import Boom from 'boom';
import uuid from 'uuid';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { createRules } from '../../rules/create_rules';
import { RulesRequest } from '../../rules/types';
import { RulesRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { createRulesSchema } from '../schemas/create_rules_schema';
import { ServerFacade } from '../../../../types';
import { readRules } from '../../rules/read_rules';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
import { transformOrError } from './utils';
import { getIndexExists } from '../../index/get_index_exists';
import { callWithRequestFactory, getIndex, transformError } from '../utils';
@ -65,8 +66,10 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute =
const actionsClient = isFunction(request.getActionsClient)
? request.getActionsClient()
: null;
if (!alertsClient || !actionsClient) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
@ -120,7 +123,17 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute =
references,
version: 1,
});
return transformOrError(createdRule);
const ruleStatuses = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 1,
sortField: 'statusDate',
sortOrder: 'desc',
search: `${createdRule.id}`,
searchFields: ['alertId'],
});
return transformOrError(createdRule, ruleStatuses.saved_objects[0]);
} catch (err) {
return transformError(err);
}

16
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.test.ts
View file

@ -20,18 +20,20 @@ import {
getDeleteBulkRequestById,
getDeleteAsPostBulkRequest,
getDeleteAsPostBulkRequestById,
getFindResultStatus,
} from '../__mocks__/request_responses';
import { ServerFacade } from '../../../../types';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { deleteRulesBulkRoute } from './delete_rules_bulk_route';
import { BulkError } from '../utils';
describe('delete_rules', () => {
let { server, alertsClient } = createMockServer();
let { server, alertsClient, savedObjectsClient } = createMockServer();
beforeEach(() => {
({ server, alertsClient } = createMockServer());
deleteRulesBulkRoute(server);
({ server, alertsClient, savedObjectsClient } = createMockServer());
deleteRulesBulkRoute((server as unknown) as ServerFacade);
});
afterEach(() => {
@ -83,6 +85,8 @@ describe('delete_rules', () => {
alertsClient.find.mockResolvedValue(getFindResult());
alertsClient.get.mockResolvedValue(getResult());
alertsClient.delete.mockResolvedValue({});
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
savedObjectsClient.delete.mockResolvedValue({});
const { payload } = await server.inject(getDeleteBulkRequest());
const parsed: BulkError[] = JSON.parse(payload);
const expected: BulkError[] = [
@ -96,14 +100,14 @@ describe('delete_rules', () => {
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
deleteRulesBulkRoute(serverWithoutActionClient);
deleteRulesBulkRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getDeleteBulkRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
deleteRulesBulkRoute(serverWithoutAlertClient);
deleteRulesBulkRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getDeleteBulkRequest());
expect(statusCode).toBe(404);
});
@ -112,7 +116,7 @@ describe('delete_rules', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
deleteRulesBulkRoute(serverWithoutActionOrAlertClient);
deleteRulesBulkRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getDeleteBulkRequest());
expect(statusCode).toBe(404);
});

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/delete_rules_bulk_route.ts
View file

@ -13,7 +13,8 @@ import { ServerFacade } from '../../../../types';
import { queryRulesBulkSchema } from '../schemas/query_rules_bulk_schema';
import { transformOrBulkError, getIdBulkError } from './utils';
import { transformBulkError } from '../utils';
import { QueryBulkRequest } from '../../rules/types';
import { QueryBulkRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
export const createDeleteRulesBulkRoute: Hapi.ServerRoute = {
method: ['POST', 'DELETE'], // allow both POST and DELETE in case their client does not support bodies in DELETE
@ -30,8 +31,10 @@ export const createDeleteRulesBulkRoute: Hapi.ServerRoute = {
async handler(request: QueryBulkRequest, headers) {
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
if (alertsClient == null || actionsClient == null) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
const rules = Promise.all(
@ -45,8 +48,18 @@ export const createDeleteRulesBulkRoute: Hapi.ServerRoute = {
id,
ruleId,
});
if (rule != null) {
const ruleStatuses = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 6,
search: rule.id,
searchFields: ['alertId'],
});
ruleStatuses.saved_objects.forEach(async obj =>
savedObjectsClient.delete(ruleStatusSavedObjectType, obj.id)
);
return transformOrBulkError(idOrRuleIdOrUnknown, rule);
} else {
return getIdBulkError({ id, ruleId });

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/delete_rules_route.test.ts
View file

@ -13,21 +13,24 @@ import {
import { deleteRulesRoute } from './delete_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getResult,
getDeleteRequest,
getFindResultWithSingleHit,
getDeleteRequestById,
getFindResultStatus,
} from '../__mocks__/request_responses';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
describe('delete_rules', () => {
let { server, alertsClient } = createMockServer();
let { server, alertsClient, savedObjectsClient } = createMockServer();
beforeEach(() => {
({ server, alertsClient } = createMockServer());
deleteRulesRoute(server);
({ server, alertsClient, savedObjectsClient } = createMockServer());
deleteRulesRoute((server as unknown) as ServerFacade);
});
afterEach(() => {
@ -39,6 +42,8 @@ describe('delete_rules', () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());
alertsClient.delete.mockResolvedValue({});
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
savedObjectsClient.delete.mockResolvedValue({});
const { statusCode } = await server.inject(getDeleteRequest());
expect(statusCode).toBe(200);
});
@ -47,6 +52,8 @@ describe('delete_rules', () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());
alertsClient.delete.mockResolvedValue({});
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
savedObjectsClient.delete.mockResolvedValue({});
const { statusCode } = await server.inject(getDeleteRequestById());
expect(statusCode).toBe(200);
});
@ -55,20 +62,22 @@ describe('delete_rules', () => {
alertsClient.find.mockResolvedValue(getFindResult());
alertsClient.get.mockResolvedValue(getResult());
alertsClient.delete.mockResolvedValue({});
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
savedObjectsClient.delete.mockResolvedValue({});
const { statusCode } = await server.inject(getDeleteRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
deleteRulesRoute(serverWithoutActionClient);
deleteRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getDeleteRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
deleteRulesRoute(serverWithoutAlertClient);
deleteRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getDeleteRequest());
expect(statusCode).toBe(404);
});
@ -77,7 +86,7 @@ describe('delete_rules', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
deleteRulesRoute(serverWithoutActionOrAlertClient);
deleteRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getDeleteRequest());
expect(statusCode).toBe(404);
});

23
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/delete_rules_route.ts
View file

@ -13,7 +13,8 @@ import { ServerFacade } from '../../../../types';
import { queryRulesSchema } from '../schemas/query_rules_schema';
import { getIdError, transformOrError } from './utils';
import { transformError } from '../utils';
import { QueryRequest } from '../../rules/types';
import { QueryRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
export const createDeleteRulesRoute: Hapi.ServerRoute = {
method: 'DELETE',
@ -31,8 +32,10 @@ export const createDeleteRulesRoute: Hapi.ServerRoute = {
const { id, rule_id: ruleId } = request.query;
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
if (alertsClient == null || actionsClient == null) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
@ -43,9 +46,19 @@ export const createDeleteRulesRoute: Hapi.ServerRoute = {
id,
ruleId,
});
if (rule != null) {
return transformOrError(rule);
const ruleStatuses = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 6,
search: rule.id,
searchFields: ['alertId'],
});
ruleStatuses.saved_objects.forEach(async obj =>
savedObjectsClient.delete(ruleStatusSavedObjectType, obj.id)
);
return transformOrError(rule, ruleStatuses.saved_objects[0]);
} else {
return getIdError({ id, ruleId });
}

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/find_rules_route.test.ts
View file

@ -13,6 +13,8 @@ import {
import { findRulesRoute } from './find_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import { getFindResult, getResult, getFindRequest } from '../__mocks__/request_responses';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
@ -21,7 +23,7 @@ describe('find_rules', () => {
beforeEach(() => {
({ server, alertsClient, actionsClient } = createMockServer());
findRulesRoute(server);
findRulesRoute((server as unknown) as ServerFacade);
});
afterEach(() => {
@ -44,14 +46,14 @@ describe('find_rules', () => {
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
findRulesRoute(serverWithoutActionClient);
findRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getFindRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
findRulesRoute(serverWithoutAlertClient);
findRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getFindRequest());
expect(statusCode).toBe(404);
});
@ -60,7 +62,7 @@ describe('find_rules', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
findRulesRoute(serverWithoutActionOrAlertClient);
findRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getFindRequest());
expect(statusCode).toBe(404);
});

24
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/find_rules_route.ts
View file

@ -8,11 +8,12 @@ import Hapi from 'hapi';
import { isFunction } from 'lodash/fp';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { findRules } from '../../rules/find_rules';
import { FindRulesRequest } from '../../rules/types';
import { FindRulesRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { findRulesSchema } from '../schemas/find_rules_schema';
import { ServerFacade } from '../../../../types';
import { transformFindAlertsOrError } from './utils';
import { transformError } from '../utils';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
export const createFindRulesRoute: Hapi.ServerRoute = {
method: 'GET',
@ -30,8 +31,10 @@ export const createFindRulesRoute: Hapi.ServerRoute = {
const { query } = request;
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
if (!alertsClient || !actionsClient) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
@ -44,7 +47,20 @@ export const createFindRulesRoute: Hapi.ServerRoute = {
sortOrder: query.sort_order,
filter: query.filter,
});
return transformFindAlertsOrError(rules);
const ruleStatuses = await Promise.all(
rules.data.map(async rule => {
const results = await savedObjectsClient.find<IRuleSavedAttributesSavedObjectAttributes>({
type: ruleStatusSavedObjectType,
perPage: 1,
sortField: 'statusDate',
sortOrder: 'desc',
search: rule.id,
searchFields: ['alertId'],
});
return results;
})
);
return transformFindAlertsOrError(rules, ruleStatuses);
} catch (err) {
return transformError(err);
}

82
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/find_rules_status_route.ts Normal file
View file

@ -0,0 +1,82 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Hapi from 'hapi';
import { isFunction, snakeCase } from 'lodash/fp';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { ServerFacade } from '../../../../types';
import { findRulesStatusesSchema } from '../schemas/find_rules_statuses_schema';
import {
FindRulesStatusesRequest,
IRuleSavedAttributesSavedObjectAttributes,
} from '../../rules/types';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
const convertToSnakeCase = (obj: IRuleSavedAttributesSavedObjectAttributes) => {
return Object.keys(obj).reduce((acc, item) => {
const newKey = snakeCase(item);
return { ...acc, [newKey]: obj[item] };
}, {});
};
export const createFindRulesStatusRoute: Hapi.ServerRoute = {
method: 'GET',
path: `${DETECTION_ENGINE_RULES_URL}/_find_statuses`,
options: {
tags: ['access:siem'],
validate: {
options: {
abortEarly: false,
},
query: findRulesStatusesSchema,
},
},
async handler(request: FindRulesStatusesRequest, headers) {
const { query } = request;
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
// build return object with ids as keys and errors as values.
/* looks like this
{
"someAlertId": [{"myerrorobject": "some error value"}, etc..],
"anotherAlertId": ...
}
*/
const statuses = await query.ids.reduce(async (acc, id) => {
const lastFiveErrorsForId = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 6,
sortField: 'statusDate',
sortOrder: 'desc',
search: id,
searchFields: ['alertId'],
});
const toDisplay =
lastFiveErrorsForId.saved_objects.length <= 5
? lastFiveErrorsForId.saved_objects
: lastFiveErrorsForId.saved_objects.slice(1);
return {
...(await acc),
[id]: toDisplay.map(errorItem => convertToSnakeCase(errorItem.attributes)),
};
}, {});
return statuses;
},
};
export const findRulesStatusesRoute = (server: ServerFacade): void => {
server.route(createFindRulesStatusRoute);
};

17
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/read_rules_route.test.ts
View file

@ -13,20 +13,23 @@ import {
import { readRulesRoute } from './read_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getResult,
getReadRequest,
getFindResultWithSingleHit,
getFindResultStatus,
} from '../__mocks__/request_responses';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
describe('read_signals', () => {
let { server, alertsClient } = createMockServer();
let { server, alertsClient, savedObjectsClient } = createMockServer();
beforeEach(() => {
({ server, alertsClient } = createMockServer());
readRulesRoute(server);
({ server, alertsClient, savedObjectsClient } = createMockServer());
readRulesRoute((server as unknown) as ServerFacade);
});
afterEach(() => {
@ -37,20 +40,21 @@ describe('read_signals', () => {
test('returns 200 when reading a single rule with a valid actionClient and alertClient', async () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { statusCode } = await server.inject(getReadRequest());
expect(statusCode).toBe(200);
});
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
readRulesRoute(serverWithoutActionClient);
readRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getReadRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
readRulesRoute(serverWithoutAlertClient);
readRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getReadRequest());
expect(statusCode).toBe(404);
});
@ -59,7 +63,7 @@ describe('read_signals', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
readRulesRoute(serverWithoutActionOrAlertClient);
readRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getReadRequest());
expect(statusCode).toBe(404);
});
@ -70,6 +74,7 @@ describe('read_signals', () => {
alertsClient.find.mockResolvedValue(getFindResult());
alertsClient.get.mockResolvedValue(getResult());
alertsClient.delete.mockResolvedValue({});
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const request: ServerInjectOptions = {
method: 'GET',
url: DETECTION_ENGINE_RULES_URL,

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/read_rules_route.ts
View file

@ -13,7 +13,8 @@ import { transformError } from '../utils';
import { readRules } from '../../rules/read_rules';
import { ServerFacade } from '../../../../types';
import { queryRulesSchema } from '../schemas/query_rules_schema';
import { QueryRequest } from '../../rules/types';
import { QueryRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
export const createReadRulesRoute: Hapi.ServerRoute = {
method: 'GET',
@ -31,8 +32,10 @@ export const createReadRulesRoute: Hapi.ServerRoute = {
const { id, rule_id: ruleId } = request.query;
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
if (!alertsClient || !actionsClient) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
try {
@ -42,7 +45,17 @@ export const createReadRulesRoute: Hapi.ServerRoute = {
ruleId,
});
if (rule != null) {
return transformOrError(rule);
const ruleStatuses = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 1,
sortField: 'statusDate',
sortOrder: 'desc',
search: rule.id,
searchFields: ['alertId'],
});
return transformOrError(rule, ruleStatuses.saved_objects[0]);
} else {
return getIdError({ id, ruleId });
}

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_bulk.test.ts
View file

@ -13,6 +13,8 @@ import {
import { updateRulesRoute } from './update_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getResult,
@ -31,7 +33,7 @@ describe('update_rules_bulk', () => {
beforeEach(() => {
jest.resetAllMocks();
({ server, alertsClient, actionsClient } = createMockServer());
updateRulesBulkRoute(server);
updateRulesBulkRoute((server as unknown) as ServerFacade);
});
describe('status codes with actionClient and alertClient', () => {
@ -71,14 +73,14 @@ describe('update_rules_bulk', () => {
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
updateRulesRoute(serverWithoutActionClient);
updateRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getUpdateBulkRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
updateRulesRoute(serverWithoutAlertClient);
updateRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getUpdateBulkRequest());
expect(statusCode).toBe(404);
});
@ -87,7 +89,7 @@ describe('update_rules_bulk', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
updateRulesRoute(serverWithoutActionOrAlertClient);
updateRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getUpdateBulkRequest());
expect(statusCode).toBe(404);
});

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.test.ts
View file

@ -13,8 +13,11 @@ import {
import { updateRulesRoute } from './update_rules_route';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getFindResult,
getFindResultStatus,
getResult,
updateActionResult,
getUpdateRequest,
@ -24,12 +27,12 @@ import {
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
describe('update_rules', () => {
let { server, alertsClient, actionsClient } = createMockServer();
let { server, alertsClient, actionsClient, savedObjectsClient } = createMockServer();
beforeEach(() => {
jest.resetAllMocks();
({ server, alertsClient, actionsClient } = createMockServer());
updateRulesRoute(server);
({ server, alertsClient, actionsClient, savedObjectsClient } = createMockServer());
updateRulesRoute((server as unknown) as ServerFacade);
});
describe('status codes with actionClient and alertClient', () => {
@ -38,6 +41,7 @@ describe('update_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { statusCode } = await server.inject(getUpdateRequest());
expect(statusCode).toBe(200);
});
@ -47,20 +51,21 @@ describe('update_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { statusCode } = await server.inject(getUpdateRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if actionClient is not available on the route', async () => {
const { serverWithoutActionClient } = createMockServerWithoutActionClientDecoration();
updateRulesRoute(serverWithoutActionClient);
updateRulesRoute((serverWithoutActionClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionClient.inject(getUpdateRequest());
expect(statusCode).toBe(404);
});
test('returns 404 if alertClient is not available on the route', async () => {
const { serverWithoutAlertClient } = createMockServerWithoutAlertClientDecoration();
updateRulesRoute(serverWithoutAlertClient);
updateRulesRoute((serverWithoutAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutAlertClient.inject(getUpdateRequest());
expect(statusCode).toBe(404);
});
@ -69,7 +74,7 @@ describe('update_rules', () => {
const {
serverWithoutActionOrAlertClient,
} = createMockServerWithoutActionOrAlertClientDecoration();
updateRulesRoute(serverWithoutActionOrAlertClient);
updateRulesRoute((serverWithoutActionOrAlertClient as unknown) as ServerFacade);
const { statusCode } = await serverWithoutActionOrAlertClient.inject(getUpdateRequest());
expect(statusCode).toBe(404);
});
@ -79,6 +84,7 @@ describe('update_rules', () => {
test('returns 400 if id is not given in either the body or the url', async () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { rule_id, ...noId } = typicalPayload();
const request: ServerInjectOptions = {
method: 'PUT',
@ -95,6 +101,7 @@ describe('update_rules', () => {
alertsClient.find.mockResolvedValue(getFindResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const request: ServerInjectOptions = {
method: 'PUT',
url: DETECTION_ENGINE_RULES_URL,
@ -109,6 +116,7 @@ describe('update_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const request: ServerInjectOptions = {
method: 'PUT',
url: DETECTION_ENGINE_RULES_URL,
@ -123,6 +131,7 @@ describe('update_rules', () => {
alertsClient.get.mockResolvedValue(getResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
savedObjectsClient.find.mockResolvedValue(getFindResultStatus());
const { type, ...noType } = typicalPayload();
const request: ServerInjectOptions = {
method: 'PUT',

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/update_rules_route.ts
View file

@ -8,11 +8,12 @@ import Hapi from 'hapi';
import { isFunction } from 'lodash/fp';
import { DETECTION_ENGINE_RULES_URL } from '../../../../../common/constants';
import { updateRules } from '../../rules/update_rules';
import { UpdateRulesRequest } from '../../rules/types';
import { UpdateRulesRequest, IRuleSavedAttributesSavedObjectAttributes } from '../../rules/types';
import { updateRulesSchema } from '../schemas/update_rules_schema';
import { ServerFacade } from '../../../../types';
import { getIdError, transformOrError } from './utils';
import { transformError } from '../utils';
import { ruleStatusSavedObjectType } from '../../rules/saved_object_mappings';
export const createUpdateRulesRoute: Hapi.ServerRoute = {
method: 'PUT',
@ -59,8 +60,10 @@ export const createUpdateRulesRoute: Hapi.ServerRoute = {
const alertsClient = isFunction(request.getAlertsClient) ? request.getAlertsClient() : null;
const actionsClient = isFunction(request.getActionsClient) ? request.getActionsClient() : null;
if (!alertsClient || !actionsClient) {
const savedObjectsClient = isFunction(request.getSavedObjectsClient)
? request.getSavedObjectsClient()
: null;
if (!alertsClient || !actionsClient || !savedObjectsClient) {
return headers.response().code(404);
}
@ -97,7 +100,17 @@ export const createUpdateRulesRoute: Hapi.ServerRoute = {
version,
});
if (rule != null) {
return transformOrError(rule);
const ruleStatuses = await savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 1,
sortField: 'statusDate',
sortOrder: 'desc',
search: rule.id,
searchFields: ['alertId'],
});
return transformOrError(rule, ruleStatuses.saved_objects[0]);
} else {
return getIdError({ id, ruleId });
}

56
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/rules/utils.ts
View file

@ -6,8 +6,17 @@
import Boom from 'boom';
import { pickBy } from 'lodash/fp';
import { SavedObject } from 'kibana/server';
import { INTERNAL_IDENTIFIER } from '../../../../../common/constants';
import { RuleAlertType, isAlertType, isAlertTypes } from '../../rules/types';
import {
RuleAlertType,
isAlertType,
isAlertTypes,
IRuleSavedAttributesSavedObjectAttributes,
isRuleStatusFindType,
isRuleStatusFindTypes,
isRuleStatusSavedObjectType,
} from '../../rules/types';
import { OutputRuleAlertRest } from '../../types';
import {
createBulkErrorObject,
@ -67,7 +76,10 @@ export const transformTags = (tags: string[]): string[] => {
// Transforms the data but will remove any null or undefined it encounters and not include
// those on the export
export const transformAlertToRule = (alert: RuleAlertType): Partial<OutputRuleAlertRest> => {
export const transformAlertToRule = (
alert: RuleAlertType,
ruleStatus?: SavedObject<IRuleSavedAttributesSavedObjectAttributes>
): Partial<OutputRuleAlertRest> => {
return pickBy<OutputRuleAlertRest>((value: unknown) => value != null, {
created_at: alert.params.createdAt,
updated_at: alert.params.updatedAt,
@ -100,6 +112,12 @@ export const transformAlertToRule = (alert: RuleAlertType): Partial<OutputRuleAl
type: alert.params.type,
threats: alert.params.threats,
version: alert.params.version,
status: ruleStatus?.attributes.status,
status_date: ruleStatus?.attributes.statusDate,
last_failure_at: ruleStatus?.attributes.lastFailureAt,
last_success_at: ruleStatus?.attributes.lastSuccessAt,
last_failure_message: ruleStatus?.attributes.lastFailureMessage,
last_success_message: ruleStatus?.attributes.lastSuccessMessage,
});
};
@ -118,18 +136,35 @@ export const transformAlertsToRules = (
return alerts.map(alert => transformAlertToRule(alert));
};
export const transformFindAlertsOrError = (findResults: { data: unknown[] }): unknown | Boom => {
if (isAlertTypes(findResults.data)) {
export const transformFindAlertsOrError = (
findResults: { data: unknown[] },
ruleStatuses?: unknown[]
): unknown | Boom => {
if (!ruleStatuses && isAlertTypes(findResults.data)) {
findResults.data = findResults.data.map(alert => transformAlertToRule(alert));
return findResults;
}
if (isAlertTypes(findResults.data) && isRuleStatusFindTypes(ruleStatuses)) {
findResults.data = findResults.data.map((alert, idx) =>
transformAlertToRule(alert, ruleStatuses[idx].saved_objects[0])
);
return findResults;
} else {
return new Boom('Internal error transforming', { statusCode: 500 });
}
};
export const transformOrError = (alert: unknown): Partial<OutputRuleAlertRest> | Boom => {
if (isAlertType(alert)) {
export const transformOrError = (
alert: unknown,
ruleStatus?: unknown
): Partial<OutputRuleAlertRest> | Boom => {
if (!ruleStatus && isAlertType(alert)) {
return transformAlertToRule(alert);
}
if (isAlertType(alert) && isRuleStatusFindType(ruleStatus)) {
return transformAlertToRule(alert, ruleStatus.saved_objects[0]);
} else if (isAlertType(alert) && isRuleStatusSavedObjectType(ruleStatus)) {
return transformAlertToRule(alert, ruleStatus);
} else {
return new Boom('Internal error transforming', { statusCode: 500 });
}
@ -137,10 +172,15 @@ export const transformOrError = (alert: unknown): Partial<OutputRuleAlertRest> |
export const transformOrBulkError = (
ruleId: string,
alert: unknown
alert: unknown,
ruleStatus?: unknown
): Partial<OutputRuleAlertRest> | BulkError => {
if (isAlertType(alert)) {
return transformAlertToRule(alert);
if (isRuleStatusFindType(ruleStatus)) {
return transformAlertToRule(alert, ruleStatus?.saved_objects[0] ?? ruleStatus);
} else {
return transformAlertToRule(alert);
}
} else {
return createBulkErrorObject({
ruleId,

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas/find_rules_statuses_schema.ts Normal file
View file

@ -0,0 +1,11 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import Joi from 'joi';
export const findRulesStatusesSchema = Joi.object({
ids: Joi.array().items(Joi.string()),
});

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/open_close_signals.test.ts
View file

@ -8,6 +8,8 @@ import { createMockServer } from '../__mocks__/_mock_server';
import { setSignalsStatusRoute } from './open_close_signals_route';
import * as myUtils from '../utils';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getSetSignalStatusByIdsRequest,
getSetSignalStatusByQueryRequest,
@ -27,7 +29,7 @@ describe('set signal status', () => {
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(() => true),
}));
setSignalsStatusRoute(server);
setSignalsStatusRoute((server as unknown) as ServerFacade);
});
describe('status on signal', () => {

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/signals/query_signals_route.test.ts
View file

@ -8,6 +8,8 @@ import { createMockServer } from '../__mocks__/_mock_server';
import { querySignalsRoute } from './query_signals_route';
import * as myUtils from '../utils';
import { ServerInjectOptions } from 'hapi';
import { ServerFacade } from '../../../../types';
import {
getSignalsQueryRequest,
getSignalsAggsQueryRequest,
@ -26,7 +28,7 @@ describe('query for signal', () => {
elasticsearch.getCluster = jest.fn(() => ({
callWithRequest: jest.fn(() => true),
}));
querySignalsRoute(server);
querySignalsRoute((server as unknown) as ServerFacade);
});
describe('query and agg on signals index', () => {

40
x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/saved_object_mappings.ts Normal file
View file

@ -0,0 +1,40 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { ElasticsearchMappingOf } from '../../../utils/typed_elasticsearch_mappings';
import { IRuleStatusAttributes } from './types';
export const ruleStatusSavedObjectType = 'siem-detection-engine-rule-status';
export const ruleStatusSavedObjectMappings: {
[ruleStatusSavedObjectType]: ElasticsearchMappingOf<IRuleStatusAttributes>;
} = {
[ruleStatusSavedObjectType]: {
properties: {
alertId: {
type: 'keyword',
},
status: {
type: 'keyword',
},
statusDate: {
type: 'date',
},
lastFailureAt: {
type: 'date',
},
lastSuccessAt: {
type: 'date',
},
lastFailureMessage: {
type: 'text',
},
lastSuccessMessage: {
type: 'text',
},
},
},
};

61
x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/types.ts
View file

@ -7,6 +7,7 @@
import { get } from 'lodash/fp';
import { Readable } from 'stream';
import { SavedObject, SavedObjectAttributes, SavedObjectsFindResponse } from 'kibana/server';
import { SIGNALS_ID } from '../../../../common/constants';
import { AlertsClient } from '../../../../../alerting/server/alerts_client';
import { ActionsClient } from '../../../../../actions/server/actions_client';
@ -40,6 +41,38 @@ export interface RuleAlertType extends Alert {
params: RuleTypeParams;
}
export interface IRuleStatusAttributes {
alertId: string; // created alert id.
statusDate: string;
lastFailureAt: string | null | undefined;
lastFailureMessage: string | null | undefined;
lastSuccessAt: string | null | undefined;
lastSuccessMessage: string | null | undefined;
status: RuleStatusString;
}
export interface IRuleSavedAttributesSavedObjectAttributes
extends IRuleStatusAttributes,
SavedObjectAttributes {}
export interface IRuleStatusSavedObject {
type: string;
id: string;
attributes: Array<SavedObject<IRuleStatusAttributes & SavedObjectAttributes>>;
references: unknown[];
updated_at: string;
version: string;
}
export interface IRuleStatusFindType {
page: number;
per_page: number;
total: number;
saved_objects: IRuleStatusSavedObject[];
}
export type RuleStatusString = 'succeeded' | 'failed' | 'going to run' | 'executing';
export interface RulesRequest extends RequestFacade {
payload: RuleAlertParamsRest;
}
@ -96,6 +129,12 @@ export interface FindRulesRequest extends Omit<RequestFacade, 'query'> {
};
}
export interface FindRulesStatusesRequest extends Omit<RequestFacade, 'query'> {
query: {
ids: string[];
};
}
export interface Clients {
alertsClient: AlertsClient;
actionsClient: ActionsClient;
@ -125,3 +164,25 @@ export const isAlertTypes = (obj: unknown[]): obj is RuleAlertType[] => {
export const isAlertType = (obj: unknown): obj is RuleAlertType => {
return get('alertTypeId', obj) === SIGNALS_ID;
};
export const isRuleStatusAttributes = (obj: unknown): obj is IRuleStatusAttributes => {
return get('lastSuccessMessage', obj) != null;
};
export const isRuleStatusSavedObjectType = (
obj: unknown
): obj is SavedObject<IRuleSavedAttributesSavedObjectAttributes> => {
return get('attributes', obj) != null;
};
export const isRuleStatusFindType = (
obj: unknown
): obj is SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes> => {
return get('saved_objects', obj) != null;
};
export const isRuleStatusFindTypes = (
obj: unknown[] | undefined
): obj is Array<SavedObjectsFindResponse<IRuleSavedAttributesSavedObjectAttributes>> => {
return obj ? obj.every(ruleStatus => isRuleStatusFindType(ruleStatus)) : false;
};

23
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/delete_all_statuses.sh Executable file
View file

@ -0,0 +1,23 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.
#
set -e
./check_env_variables.sh
# Example: ./delete_all_statuses.sh
# https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html
curl -s -k \
-H "Content-Type: application/json" \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-X POST ${ELASTICSEARCH_URL}/${KIBANA_INDEX}*/_delete_by_query \
--data '{
"query": {
"exists": { "field": "siem-detection-engine-rule-status" }
}
}' \
| jq .

17
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/find_rules_statuses_by_ids.sh Executable file
View file

@ -0,0 +1,17 @@
#!/bin/sh
#
# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
# or more contributor license agreements. Licensed under the Elastic License;
# you may not use this file except in compliance with the Elastic License.
#
set -e
./check_env_variables.sh
# Example: ./find_rules_statuses_by_ids.sh '["12345","6789abc"]'
curl -g -k \
-u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
-X GET "${KIBANA_URL}${SPACE_URL}/api/detection_engine/rules/_find_statuses?ids=$1" \
| jq .

1
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh
View file

@ -12,5 +12,6 @@ set -e
./delete_all_actions.sh
./delete_all_alerts.sh
./delete_all_alert_tasks.sh
./delete_all_statuses.sh
./delete_signal_index.sh
./post_signal_index.sh

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/query_with_errors.json Normal file
View file

@ -0,0 +1,12 @@
{
"name": "Query with a rule_id that causes an error",
"description": "Query with a rule_id that acts like an external id",
"rule_id": "query-rule-id-errors-2",
"risk_score": 1,
"interval": "30s",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root badstringcauseserror user.name: admin"
}

258
x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -19,6 +19,8 @@ import { searchAfterAndBulkCreate } from './search_after_bulk_create';
import { getFilter } from './get_filter';
import { SignalRuleAlertTypeDefinition } from './types';
import { getGapBetweenRuns } from './utils';
import { ruleStatusSavedObjectType } from '../rules/saved_object_mappings';
import { IRuleSavedAttributesSavedObjectAttributes } from '../rules/types';
export const signalRulesAlertType = ({
logger,
@ -75,6 +77,46 @@ export const signalRulesAlertType = ({
} = params;
// TODO: Remove this hard extraction of name once this is fixed: https://github.com/elastic/kibana/issues/50522
const savedObject = await services.savedObjectsClient.get('alert', alertId);
const ruleStatusSavedObjects = await services.savedObjectsClient.find<
IRuleSavedAttributesSavedObjectAttributes
>({
type: ruleStatusSavedObjectType,
perPage: 6, // 0th element is current status, 1-5 is last 5 failures.
sortField: 'statusDate',
sortOrder: 'desc',
search: `${alertId}`,
searchFields: ['alertId'],
});
let currentStatusSavedObject;
if (ruleStatusSavedObjects.saved_objects.length === 0) {
// create
const date = new Date().toISOString();
currentStatusSavedObject = await services.savedObjectsClient.create<
IRuleSavedAttributesSavedObjectAttributes
>(ruleStatusSavedObjectType, {
alertId, // do a search for this id.
statusDate: date,
status: 'executing',
lastFailureAt: null,
lastSuccessAt: null,
lastFailureMessage: null,
lastSuccessMessage: null,
});
} else {
// update 0th to executing.
currentStatusSavedObject = ruleStatusSavedObjects.saved_objects[0];
const sDate = new Date().toISOString();
currentStatusSavedObject.attributes.status = 'executing';
currentStatusSavedObject.attributes.statusDate = sDate;
await services.savedObjectsClient.update(
ruleStatusSavedObjectType,
currentStatusSavedObject.id,
{
...currentStatusSavedObject.attributes,
}
);
}
const name: string = savedObject.attributes.name;
const tags: string[] = savedObject.attributes.tags;
@ -98,77 +140,169 @@ export const signalRulesAlertType = ({
DEFAULT_SEARCH_AFTER_PAGE_SIZE <= params.maxSignals
? DEFAULT_SEARCH_AFTER_PAGE_SIZE
: params.maxSignals;
const inputIndex = await getInputIndex(services, version, index);
const esFilter = await getFilter({
type,
filters,
language,
query,
savedId,
services,
index: inputIndex,
});
const noReIndex = buildEventsSearchQuery({
index: inputIndex,
from,
to,
filter: esFilter,
size: searchAfterSize,
searchAfterSortId: undefined,
});
try {
logger.debug(
`Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
logger.debug(
`[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const noReIndexResult = await services.callCluster('search', noReIndex);
if (noReIndexResult.hits.total.value !== 0) {
logger.info(
`Found ${
noReIndexResult.hits.total.value
} signals from the indexes of "[${inputIndex.join(
', '
)}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
);
}
const bulkIndexResult = await searchAfterAndBulkCreate({
someResult: noReIndexResult,
ruleParams: params,
const inputIndex = await getInputIndex(services, version, index);
const esFilter = await getFilter({
type,
filters,
language,
query,
savedId,
services,
logger,
id: alertId,
signalsIndex: outputIndex,
filter: esFilter,
name,
createdBy,
updatedBy,
interval,
enabled,
pageSize: searchAfterSize,
tags,
index: inputIndex,
});
if (bulkIndexResult) {
const noReIndex = buildEventsSearchQuery({
index: inputIndex,
from,
to,
filter: esFilter,
size: searchAfterSize,
searchAfterSortId: undefined,
});
try {
logger.debug(
`Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
`Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
} else {
logger.debug(
`[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const noReIndexResult = await services.callCluster('search', noReIndex);
if (noReIndexResult.hits.total.value !== 0) {
logger.info(
`Found ${
noReIndexResult.hits.total.value
} signals from the indexes of "[${inputIndex.join(
', '
)}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
);
}
const bulkIndexResult = await searchAfterAndBulkCreate({
someResult: noReIndexResult,
ruleParams: params,
services,
logger,
id: alertId,
signalsIndex: outputIndex,
filter: esFilter,
name,
createdBy,
updatedBy,
interval,
enabled,
pageSize: searchAfterSize,
tags,
});
if (bulkIndexResult) {
logger.debug(
`Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const sDate = new Date().toISOString();
currentStatusSavedObject.attributes.status = 'succeeded';
currentStatusSavedObject.attributes.statusDate = sDate;
currentStatusSavedObject.attributes.lastSuccessAt = sDate;
currentStatusSavedObject.attributes.lastSuccessMessage = 'succeeded';
await services.savedObjectsClient.update(
ruleStatusSavedObjectType,
currentStatusSavedObject.id,
{
...currentStatusSavedObject.attributes,
}
);
} else {
logger.error(
`Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const sDate = new Date().toISOString();
currentStatusSavedObject.attributes.status = 'failed';
currentStatusSavedObject.attributes.statusDate = sDate;
currentStatusSavedObject.attributes.lastFailureAt = sDate;
currentStatusSavedObject.attributes.lastFailureMessage = `Bulk Indexing signals failed. Check logs for further details \nRule name: "${name}"\nid: "${alertId}"\nrule_id: "${ruleId}"\n`;
// current status is failing
await services.savedObjectsClient.update(
ruleStatusSavedObjectType,
currentStatusSavedObject.id,
{
...currentStatusSavedObject.attributes,
}
);
// create new status for historical purposes
await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
...currentStatusSavedObject.attributes,
});
if (ruleStatusSavedObjects.saved_objects.length >= 6) {
// delete fifth status and prepare to insert a newer one.
const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
await toDelete.forEach(async item =>
services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
);
}
}
} catch (err) {
// TODO: Error handling and writing of errors into a signal that has error
// handling/conditions
logger.error(
`Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const sDate = new Date().toISOString();
currentStatusSavedObject.attributes.status = 'failed';
currentStatusSavedObject.attributes.statusDate = sDate;
currentStatusSavedObject.attributes.lastFailureAt = sDate;
currentStatusSavedObject.attributes.lastFailureMessage = err.message;
// current status is failing
await services.savedObjectsClient.update(
ruleStatusSavedObjectType,
currentStatusSavedObject.id,
{
...currentStatusSavedObject.attributes,
}
);
// create new status for historical purposes
await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
...currentStatusSavedObject.attributes,
});
if (ruleStatusSavedObjects.saved_objects.length >= 6) {
// delete fifth status and prepare to insert a newer one.
const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
await toDelete.forEach(async item =>
services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
);
}
}
} catch (exception) {
logger.error(
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${exception.message}`
);
const sDate = new Date().toISOString();
currentStatusSavedObject.attributes.status = 'failed';
currentStatusSavedObject.attributes.statusDate = sDate;
currentStatusSavedObject.attributes.lastFailureAt = sDate;
currentStatusSavedObject.attributes.lastFailureMessage = exception.message;
// current status is failing
await services.savedObjectsClient.update(
ruleStatusSavedObjectType,
currentStatusSavedObject.id,
{
...currentStatusSavedObject.attributes,
}
);
// create new status for historical purposes
await services.savedObjectsClient.create(ruleStatusSavedObjectType, {
...currentStatusSavedObject.attributes,
});
if (ruleStatusSavedObjects.saved_objects.length >= 6) {
// delete fifth status and prepare to insert a newer one.
const toDelete = ruleStatusSavedObjects.saved_objects.slice(5);
await toDelete.forEach(async item =>
services.savedObjectsClient.delete(ruleStatusSavedObjectType, item.id)
);
}
} catch (err) {
// TODO: Error handling and writing of errors into a signal that has error
// handling/conditions
logger.error(
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}" message: ${err.message}`
);
}
},
};

41
x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts
View file

@ -5,6 +5,7 @@
*/
import { esFilters } from '../../../../../../../src/plugins/data/server';
import { IRuleStatusAttributes } from './rules/types';
export type PartialFilter = Partial<esFilters.Filter>;
@ -65,18 +66,34 @@ export type RuleAlertParamsRest = Omit<
| 'outputIndex'
| 'updatedAt'
| 'createdAt'
> & {
rule_id: RuleAlertParams['ruleId'];
false_positives: RuleAlertParams['falsePositives'];
saved_id: RuleAlertParams['savedId'];
timeline_id: RuleAlertParams['timelineId'];
timeline_title: RuleAlertParams['timelineTitle'];
max_signals: RuleAlertParams['maxSignals'];
risk_score: RuleAlertParams['riskScore'];
output_index: RuleAlertParams['outputIndex'];
created_at: RuleAlertParams['createdAt'];
updated_at: RuleAlertParams['updatedAt'];
};
> &
Omit<
IRuleStatusAttributes,
| 'status'
| 'alertId'
| 'statusDate'
| 'lastFailureAt'
| 'lastSuccessAt'
| 'lastSuccessMessage'
| 'lastFailureMessage'
> & {
rule_id: RuleAlertParams['ruleId'];
false_positives: RuleAlertParams['falsePositives'];
saved_id: RuleAlertParams['savedId'];
timeline_id: RuleAlertParams['timelineId'];
timeline_title: RuleAlertParams['timelineTitle'];
max_signals: RuleAlertParams['maxSignals'];
risk_score: RuleAlertParams['riskScore'];
output_index: RuleAlertParams['outputIndex'];
created_at: RuleAlertParams['createdAt'];
updated_at: RuleAlertParams['updatedAt'];
status?: IRuleStatusAttributes['status'] | undefined;
status_date?: IRuleStatusAttributes['statusDate'] | undefined;
last_failure_at?: IRuleStatusAttributes['lastFailureAt'] | undefined;
last_success_at?: IRuleStatusAttributes['lastSuccessAt'] | undefined;
last_failure_message?: IRuleStatusAttributes['lastFailureMessage'] | undefined;
last_success_message?: IRuleStatusAttributes['lastSuccessMessage'] | undefined;
};
export type OutputRuleAlertRest = RuleAlertParamsRest & {
id: string;

2
x-pack/legacy/plugins/siem/server/saved_objects.ts
View file

@ -12,10 +12,12 @@ import {
timelineSavedObjectType,
timelineSavedObjectMappings,
} from './lib/timeline/saved_object_mappings';
import { ruleStatusSavedObjectMappings } from './lib/detection_engine/rules/saved_object_mappings';
export { noteSavedObjectType, pinnedEventSavedObjectType, timelineSavedObjectType };
export const savedObjectMappings = {
...timelineSavedObjectMappings,
...noteSavedObjectMappings,
...pinnedEventSavedObjectMappings,
...ruleStatusSavedObjectMappings,
};

2
x-pack/legacy/plugins/siem/server/types.ts
View file

@ -12,6 +12,7 @@ export interface ServerFacade {
alerting?: Legacy.Server['plugins']['alerting'];
elasticsearch: Legacy.Server['plugins']['elasticsearch'];
spaces: Legacy.Server['plugins']['spaces'];
savedObjects: Legacy.Server['savedObjects']['SavedObjectsClient'];
};
route: Legacy.Server['route'];
}
@ -20,6 +21,7 @@ export interface RequestFacade {
auth: Legacy.Request['auth'];
getAlertsClient?: Legacy.Request['getAlertsClient'];
getActionsClient?: Legacy.Request['getActionsClient'];
getSavedObjectsClient?: Legacy.Request['getSavedObjectsClient'];
headers: Legacy.Request['headers'];
method: Legacy.Request['method'];
params: Legacy.Request['params'];

2
x-pack/legacy/plugins/siem/server/utils/typed_elasticsearch_mappings.ts
View file

@ -17,7 +17,7 @@ export type ElasticsearchMappingOf<Type> = Type extends string | string[]
: never;
export interface ElasticsearchStringFieldMapping {
type: 'keyword' | 'text';
type: 'keyword' | 'text' | 'date';
}
export interface ElasticsearchBooleanFieldMapping {