From 6f7ca4a4f43e1f8f105f2cd8a4ae28790a8c51b1 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Wed, 13 Nov 2019 17:13:44 -0700 Subject: [PATCH] [SIEM][Detection Engine] REST API Additions (#50514) ## Summary Added these to the create and update API: * tags - Array string type (default []) * False positives - Array string type (default []) * immutable - boolean (default -- false) Added these instructions to the READM.md * Added "brew install jq" for all the scripts to work in the scripts folder in README.md * Added tip for debug logging Changed these shell scripts: * Removed the delete all api keys from the hard_reset script * Changed the script for converting to rules to use the new immutable flag. Testing * Added unit tests for new schema types * Added ad-hoc test for scripts * Test ran through the saved searches ### Checklist Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR. ~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~ ~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~ ~~- [ ] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials~~ - [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios ~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~ ### For maintainers ~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ ~~- [ ] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~ --- .../convert_saved_search_to_signals.js | 2 + .../server/lib/detection_engine/README.md | 23 +++ .../detection_engine/alerts/create_signals.ts | 15 ++ .../alerts/signals_alert_type.ts | 10 +- .../lib/detection_engine/alerts/types.ts | 9 +- .../detection_engine/alerts/update_signals.ts | 6 + .../routes/create_signals_route.ts | 7 + .../detection_engine/routes/schemas.test.ts | 132 ++++++++++++++++++ .../lib/detection_engine/routes/schemas.ts | 9 ++ .../routes/update_signals_route.ts | 6 + .../detection_engine/scripts/hard_reset.sh | 1 - .../scripts/signals/root_or_admin_9.json | 18 +++ .../signals/root_or_admin_update_2.json | 17 +++ 13 files changed, 246 insertions(+), 9 deletions(-) create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_9.json create mode 100644 x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_2.json diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js index 9b2a38e372ae..e6dcefd6fb4f 100644 --- a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js +++ b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js @@ -33,6 +33,7 @@ const SEVERITY = 'low'; const TYPE = 'query'; const FROM = 'now-6m'; const TO = 'now'; +const IMMUTABLE = true; const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*']; const walk = dir => { @@ -119,6 +120,7 @@ async function main() { const outputMessage = { id: fileToWrite, description: description || title, + immutable: IMMUTABLE, index: INDEX, interval: INTERVAL, name: title, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md index 5ac2a7a2d306..2c635a17ef58 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md @@ -9,6 +9,14 @@ Since there is no UI yet and a lot of backend areas that are not created, you should install the kbn-action and kbn-alert project from here: https://github.com/pmuellr/kbn-action +The scripts rely on CURL and jq, ensure both of these are installed: + +```sh +brew update +brew install curl +brew install jq +``` + Open up your .zshrc/.bashrc and add these lines with the variables filled in: ``` export ELASTICSEARCH_USERNAME=${user} @@ -127,3 +135,18 @@ created which should update once every 5 minutes at this point. Also add the `.siem-signals-${your user id}` as a kibana index for Maps to be able to see the signals + +Optionally you can add these debug statements to your `kibana.dev.yml` to see more information when running the detection +engine + +```sh +logging.verbose: true +logging.events: + { + log: ['siem', 'info', 'warning', 'error', 'fatal'], + request: ['info', 'warning', 'error', 'fatal'], + error: '*', + ops: __no-ops__, + } +``` + diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts index 038effbd3008..73e3d96f5332 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts @@ -15,8 +15,10 @@ export const updateIfIdExists = async ({ actionsClient, description, enabled, + falsePositives, filter, from, + immutable, query, language, savedId, @@ -28,6 +30,7 @@ export const updateIfIdExists = async ({ name, severity, size, + tags, to, type, references, @@ -38,8 +41,10 @@ export const updateIfIdExists = async ({ actionsClient, description, enabled, + falsePositives, filter, from, + immutable, query, language, savedId, @@ -51,6 +56,7 @@ export const updateIfIdExists = async ({ name, severity, size, + tags, to, type, references, @@ -70,6 +76,7 @@ export const createSignals = async ({ actionsClient, description, enabled, + falsePositives, filter, from, query, @@ -77,12 +84,14 @@ export const createSignals = async ({ savedId, filters, id, + immutable, index, interval, maxSignals, name, severity, size, + tags, to, type, references, @@ -93,6 +102,7 @@ export const createSignals = async ({ actionsClient, description, enabled, + falsePositives, filter, from, query, @@ -100,12 +110,14 @@ export const createSignals = async ({ savedId, filters, id, + immutable, index, interval, maxSignals, name, severity, size, + tags, to, type, references, @@ -132,14 +144,17 @@ export const createSignals = async ({ description, id, index, + falsePositives, from, filter, + immutable, query, language, savedId, filters, maxSignals, severity, + tags, to, type, references, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts index 6673b7bd1423..44175360f80b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts @@ -24,9 +24,11 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp validate: { params: schema.object({ description: schema.string(), + falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }), from: schema.string(), filter: schema.nullable(schema.object({}, { allowUnknowns: true })), id: schema.string(), + immutable: schema.boolean({ defaultValue: false }), index: schema.arrayOf(schema.string()), language: schema.nullable(schema.string()), savedId: schema.nullable(schema.string()), @@ -34,6 +36,7 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))), maxSignals: schema.number({ defaultValue: 100 }), severity: schema.string(), + tags: schema.arrayOf(schema.string(), { defaultValue: [] }), to: schema.string(), type: schema.string(), references: schema.arrayOf(schema.string(), { defaultValue: [] }), @@ -135,13 +138,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp // handling/conditions logger.error(`Error from signal rule "${id}", ${err.message}`); } - - // TODO: Schedule and fire any and all actions configured for the signals rule - // such as email/slack/etc... Note you will not be able to save in-memory state - // without calling this at least once but we are not using in-memory state at the moment. - // Schedule the default action which is nothing if it's a plain signal. - // const instance = services.alertInstanceFactory('siem-signals'); - // instance.scheduleActions('default'); }, }; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts index 96a319c944bd..7edc60618c25 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts @@ -24,9 +24,11 @@ export type PartialFilter = Partial; export interface SignalAlertParams { description: string; enabled: boolean; + falsePositives: string[]; filter: Record | undefined; filters: PartialFilter[] | undefined; from: string; + immutable: boolean; index: string[]; interval: string; id: string; @@ -38,11 +40,16 @@ export interface SignalAlertParams { savedId: string | undefined; severity: string; size: number | undefined; + tags: string[]; to: string; type: 'filter' | 'query' | 'saved_query'; } -export type SignalAlertParamsRest = Omit & { +export type SignalAlertParamsRest = Omit< + SignalAlertParams, + 'falsePositives' | 'maxSignals' | 'saved_id' +> & { + false_positives: SignalAlertParams['falsePositives']; saved_id: SignalAlertParams['savedId']; max_signals: SignalAlertParams['maxSignals']; }; diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts index 422d9123a286..b4639ee5b9ee 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_signals.ts @@ -45,6 +45,7 @@ export const updateSignal = async ({ alertsClient, actionsClient, // TODO: Use this whenever we add feature support for different action types description, + falsePositives, enabled, query, language, @@ -52,12 +53,14 @@ export const updateSignal = async ({ filters, filter, from, + immutable, id, index, interval, maxSignals, name, severity, + tags, to, type, references, @@ -78,8 +81,10 @@ export const updateSignal = async ({ }, { description, + falsePositives, filter, from, + immutable, query, language, savedId, @@ -87,6 +92,7 @@ export const updateSignal = async ({ index, maxSignals, severity, + tags, to, type, references, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts index f1fc159ffeae..d3474f214194 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts @@ -26,8 +26,11 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = { const { description, enabled, + // eslint-disable-next-line @typescript-eslint/camelcase + false_positives: falsePositives, filter, from, + immutable, query, language, // eslint-disable-next-line @typescript-eslint/camelcase @@ -41,6 +44,7 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = { name, severity, size, + tags, to, type, references, @@ -58,8 +62,10 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = { actionsClient, description, enabled, + falsePositives, filter, from, + immutable, query, language, savedId, @@ -71,6 +77,7 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = { name, severity, size, + tags, to, type, references, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts index 1045826bf488..35432c2bf56b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts @@ -563,6 +563,138 @@ describe('update_signals', () => { }).error ).toBeFalsy(); }); + + test('You can optionally send in an array of tags', () => { + expect( + createSignalsSchema.validate>({ + id: 'rule-1', + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + tags: ['tag_1', 'tag_2'], + }).error + ).toBeFalsy(); + }); + + test('You cannot send in an array of tags that are numbers', () => { + expect( + createSignalsSchema.validate< + Partial> & { tags: number[] } + >({ + id: 'rule-1', + description: 'some description', + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + tags: [0, 1, 2], + }).error + ).toBeTruthy(); + }); + + test('You can optionally send in an array of false positives', () => { + expect( + createSignalsSchema.validate>({ + id: 'rule-1', + description: 'some description', + false_positives: ['false_1', 'false_2'], + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + }).error + ).toBeFalsy(); + }); + + test('You cannot send in an array of false positives that are numbers', () => { + expect( + createSignalsSchema.validate< + Partial> & { false_positives: number[] } + >({ + id: 'rule-1', + description: 'some description', + false_positives: [5, 4], + from: 'now-5m', + to: 'now', + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + }).error + ).toBeTruthy(); + }); + + test('You can optionally set the immutable to be true', () => { + expect( + createSignalsSchema.validate>({ + id: 'rule-1', + description: 'some description', + from: 'now-5m', + to: 'now', + immutable: true, + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + }).error + ).toBeFalsy(); + }); + + test('You cannot set the immutable to be a number', () => { + expect( + createSignalsSchema.validate< + Partial> & { immutable: number } + >({ + id: 'rule-1', + description: 'some description', + from: 'now-5m', + to: 'now', + immutable: 5, + index: ['index-1'], + name: 'some-name', + severity: 'severity', + interval: '5m', + type: 'query', + references: ['index-1'], + query: 'some query', + language: 'kuery', + max_signals: 1, + }).error + ).toBeTruthy(); + }); }); describe('update signals schema', () => { diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts index 374105331672..ba2c74b8bf0a 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts @@ -9,9 +9,11 @@ import Joi from 'joi'; /* eslint-disable @typescript-eslint/camelcase */ const description = Joi.string(); const enabled = Joi.boolean(); +const false_positives = Joi.array().items(Joi.string()); const filter = Joi.object(); const filters = Joi.array(); const from = Joi.string(); +const immutable = Joi.boolean(); const id = Joi.string(); const index = Joi.array() .items(Joi.string()) @@ -35,6 +37,7 @@ const page = Joi.number() .min(1) .default(1); const sort_field = Joi.string(); +const tags = Joi.array().items(Joi.string()); const fields = Joi.array() .items(Joi.string()) .single(); @@ -43,10 +46,12 @@ const fields = Joi.array() export const createSignalsSchema = Joi.object({ description: description.required(), enabled: enabled.default(true), + false_positives: false_positives.default([]), filter: filter.when('type', { is: 'filter', then: Joi.required(), otherwise: Joi.forbidden() }), filters: filters.when('type', { is: 'query', then: Joi.optional(), otherwise: Joi.forbidden() }), from: from.required(), id: id.required(), + immutable: immutable.default(false), index: index.required(), interval: interval.default('5m'), query: query.when('type', { is: 'query', then: Joi.required(), otherwise: Joi.forbidden() }), @@ -63,6 +68,7 @@ export const createSignalsSchema = Joi.object({ max_signals: max_signals.default(100), name: name.required(), severity: severity.required(), + tags: tags.default([]), to: to.required(), type: type.required(), references: references.default([]), @@ -71,10 +77,12 @@ export const createSignalsSchema = Joi.object({ export const updateSignalSchema = Joi.object({ description, enabled, + false_positives, filter: filter.when('type', { is: 'filter', then: Joi.optional(), otherwise: Joi.forbidden() }), filters: filters.when('type', { is: 'query', then: Joi.optional(), otherwise: Joi.forbidden() }), from, id, + immutable, index, interval, query: query.when('type', { is: 'query', then: Joi.optional(), otherwise: Joi.forbidden() }), @@ -91,6 +99,7 @@ export const updateSignalSchema = Joi.object({ max_signals, name, severity, + tags, to, type, references, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts index 8248d1505143..a1c5af1158a4 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts @@ -34,8 +34,10 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = { const { description, enabled, + false_positives: falsePositives, filter, from, + immutable, query, language, // eslint-disable-next-line @typescript-eslint/camelcase @@ -49,6 +51,7 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = { name, severity, size, + tags, to, type, references, @@ -65,8 +68,10 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = { actionsClient, description, enabled, + falsePositives, filter, from, + immutable, query, language, savedId, @@ -78,6 +83,7 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = { name, severity, size, + tags, to, type, references, diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh index 9c37e93831cc..ee8fa18e1234 100755 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/hard_reset.sh @@ -14,4 +14,3 @@ set -e ./delete_all_alert_tasks.sh ./delete_signal_index.sh ./put_signal_index.sh -./delete_all_api_keys.sh diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_9.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_9.json new file mode 100644 index 000000000000..e7c348a98a7f --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_9.json @@ -0,0 +1,18 @@ +{ + "id": "rule-9", + "description": "Detecting root and admin users", + "index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"], + "interval": "5m", + "name": "Detect Root/Admin Users", + "severity": "high", + "type": "query", + "from": "now-6m", + "to": "now", + "query": "user.name: root or user.name: admin", + "language": "kuery", + "enabled": false, + "tags": ["tag_1", "tag_2"], + "false_positives": ["false_1", "false_2"], + "immutable": true, + "references": ["http://www.example.com", "https://ww.example.com"] +} diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_2.json b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_2.json new file mode 100644 index 000000000000..6b99e54d6a9b --- /dev/null +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/signals/root_or_admin_update_2.json @@ -0,0 +1,17 @@ +{ + "id": "rule-1", + "description": "Changed Description of only detecting root user", + "index": ["auditbeat-*"], + "interval": "50m", + "name": "A different name", + "severity": "high", + "type": "query", + "false_positives": ["false_update_1", "false_update_2"], + "from": "now-6m", + "immutable": true, + "tags": ["some other tag for you"], + "to": "now-5m", + "query": "user.name: root", + "language": "kuery", + "references": ["https://update1.example.com", "https://update2.example.com"] +}