diff --git a/x-pack/plugins/event_log/README.md b/x-pack/plugins/event_log/README.md index 2656718cc15e..2272341c65f5 100644 --- a/x-pack/plugins/event_log/README.md +++ b/x-pack/plugins/event_log/README.md @@ -6,13 +6,23 @@ actitivies. ## Overview This plugin provides a persistent log of "events" that can be used by other -plugins to record their processing, for later acccess. Currently it's only -used by the alerts and actions plugins. +plugins to record their processing, for later accces. It is used by: -The "events" are ECS documents, with some custom properties for Kibana, and -alerting-specific properties within those Kibana properties. The number of -ECS fields is limited today, but can be extended fairly easily. We are being -conservative in adding new fields though, to help prevent indexing explosions. +- `alerting` and `actions` plugins +- [work in progress] `security_solution` (detection rules execution log) + +The "events" are [ECS documents](https://www.elastic.co/guide/en/ecs/current/index.html) +containing both standard ECS fields and some custom fields for Kibana. + +- Standard fields are those which are defined in the ECS specification. + Examples: `@timestamp`, `message`, `event.provider`. The number of ECS fields + supported in Event Log is limited today, but can be extended fairly easily. + We are being conservative in adding new fields though, to help prevent + indexing explosions. +- Custom fields are not part of the ECS spec. We defined a top-level `kibana` + field set where we have some Kibana-specific fields like `kibana.server_uuid` + and `kibana.saved_objects`. Plugins added a few custom fields as well, + for example `kibana.alerting` field set. A client API is available for other plugins to: @@ -47,16 +57,25 @@ The structure of the event documents can be seen in the generated via a script when the structure changes. See the [README.md](generated/README.md) for how to change the document structure. -Below is an document in the expected structure, with descriptions of the fields: +Below is a document in the expected structure, with descriptions of the fields: ```js { + // Base ECS fields. + // https://www.elastic.co/guide/en/ecs/current/ecs-base.html "@timestamp": "ISO date", tags: ["tags", "here"], message: "message for humans here", + + // ECS version. This is set by the Event Log and should not be specified + // by a client of Event Log. + // https://www.elastic.co/guide/en/ecs/current/ecs-ecs.html ecs: { version: "version of ECS used by the event log", }, + + // Event fields. All of them are supported. + // https://www.elastic.co/guide/en/ecs/current/ecs-event.html event: { provider: "see below", action: "see below", @@ -65,19 +84,44 @@ Below is an document in the expected structure, with descriptions of the fields: end: "ISO date of end time for events that capture a duration", outcome: "success | failure, for events that indicate an outcome", reason: "additional detail on failure outcome", + // etc }, + + // Error fields. All of them are supported. + // https://www.elastic.co/guide/en/ecs/current/ecs-error.html error: { message: "an error message, usually associated with outcome: failure", + // etc }, + + // Log fields. Only a subset is supported. + // https://www.elastic.co/guide/en/ecs/current/ecs-log.html + log: { + level: "info | warning | any log level keyword you need", + logger: "name of the logger", + }, + + // Rule fields. All of them are supported. + // https://www.elastic.co/guide/en/ecs/current/ecs-rule.html + rule: { + author: ["Elastic"], + id: "a823fd56-5467-4727-acb1-66809737d943", + // etc + }, + + // User fields. Only user.name is supported. + // https://www.elastic.co/guide/en/ecs/current/ecs-user.html user: { name: "name of Kibana user", }, - kibana: { // custom ECS field + + // Custom fields that are not part of ECS. + kibana: { server_uuid: "UUID of kibana server, for diagnosing multi-Kibana scenarios", alerting: { instance_id: "alert instance id, for relevant documents", action_group_id: "alert action group, for relevant documents", - action_subgroup_id: "alert action subgroup, for relevant documents", + action_subgroup: "alert action subgroup, for relevant documents", status: "overall alert status, after alert execution", }, saved_objects: [ @@ -363,3 +407,14 @@ yarn test:jest x-pack/plugins/event_log --watch See: [`x-pack/test/plugin_api_integration/test_suites/event_log`](https://github.com/elastic/kibana/tree/master/x-pack/test/plugin_api_integration/test_suites/event_log). +To develop integration tests, first start the test server from the root of the repo: + +```sh +node scripts/functional_tests_server --config x-pack/test/plugin_api_integration/config.ts +``` + +Then start the test runner: + +```sh +node scripts/functional_test_runner --config x-pack/test/plugin_api_integration/config.ts --include x-pack/test/plugin_api_integration/test_suites/event_log/index.ts +``` diff --git a/x-pack/plugins/event_log/generated/README.md b/x-pack/plugins/event_log/generated/README.md index 347f5743c6d6..a6bc24852651 100644 --- a/x-pack/plugins/event_log/generated/README.md +++ b/x-pack/plugins/event_log/generated/README.md @@ -1,11 +1,26 @@ -The files in this directory were generated by manually running the script -../scripts/create-schemas.js from the root directory of the repository. +# Generating event schema -These files should not be edited by hand. +The files in this directory were generated by manually running the script +`../scripts/create-schemas.js` from the root directory of the repository. + +**These files should not be edited by hand.** Please follow the following steps: -1. clone the [ECS](https://github.com/elastic/ecs) repo locally so that it resides along side your kibana repo, and checkout the ECS version you wish to support (for example, the `1.6` branch, for version 1.6) -2. In the `x-pack/plugins/event_log/scripts/mappings.js` file you'll want to make th efollowing changes: - 1. Update `EcsKibanaExtensionsMappings` to include the mapping of the fields you wish to add. - 2. Update `EcsEventLogProperties` to include the fields in the generated mappings.json. -3. cd to the `kibana` root folder and run: `node ./x-pack/plugins/event_log/scripts/create_schemas.js` + +1. Clone the [ECS](https://github.com/elastic/ecs) repo locally so that it + resides along side your kibana repo, and checkout the ECS version you wish to + support (for example, the `1.8` branch, for version 1.8). + +2. In the `x-pack/plugins/event_log/scripts/mappings.js` file you'll want to + make the following changes: + - Update `EcsCustomPropertyMappings` to include the mapping of the custom + fields you wish to add. + - Update `EcsPropertiesToGenerate` to include the fields in the generated + `mappings.json`. + - Make sure to list all array fields in `EcsEventLogMultiValuedProperties`. + +3. Cd to the `kibana` root folder and run: + + ```sh + node ./x-pack/plugins/event_log/scripts/create_schemas.js + ``` diff --git a/x-pack/plugins/event_log/generated/mappings.json b/x-pack/plugins/event_log/generated/mappings.json index 629c4af56796..f2515d0a6a8f 100644 --- a/x-pack/plugins/event_log/generated/mappings.json +++ b/x-pack/plugins/event_log/generated/mappings.json @@ -4,6 +4,10 @@ "@timestamp": { "type": "date" }, + "message": { + "norms": false, + "type": "text" + }, "tags": { "ignore_above": 1024, "type": "keyword", @@ -11,10 +15,6 @@ "isArray": "true" } }, - "message": { - "norms": false, - "type": "text" - }, "ecs": { "properties": { "version": { @@ -23,40 +23,197 @@ } } }, + "error": { + "properties": { + "code": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "message": { + "norms": false, + "type": "text" + }, + "stack_trace": { + "doc_values": false, + "fields": { + "text": { + "norms": false, + "type": "text" + } + }, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, "event": { "properties": { "action": { "ignore_above": 1024, "type": "keyword" }, - "provider": { + "category": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "code": { "ignore_above": 1024, "type": "keyword" }, - "start": { + "created": { "type": "date" }, + "dataset": { + "ignore_above": 1024, + "type": "keyword" + }, "duration": { "type": "long" }, "end": { "type": "date" }, + "hash": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "ingested": { + "type": "date" + }, + "kind": { + "ignore_above": 1024, + "type": "keyword" + }, + "module": { + "ignore_above": 1024, + "type": "keyword" + }, + "original": { + "doc_values": false, + "ignore_above": 1024, + "index": false, + "type": "keyword" + }, "outcome": { "ignore_above": 1024, "type": "keyword" }, + "provider": { + "ignore_above": 1024, + "type": "keyword" + }, "reason": { "ignore_above": 1024, "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "risk_score": { + "type": "float" + }, + "risk_score_norm": { + "type": "float" + }, + "sequence": { + "type": "long" + }, + "severity": { + "type": "long" + }, + "start": { + "type": "date" + }, + "timezone": { + "ignore_above": 1024, + "type": "keyword" + }, + "type": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "url": { + "ignore_above": 1024, + "type": "keyword" } } }, - "error": { + "log": { "properties": { - "message": { - "norms": false, - "type": "text" + "level": { + "ignore_above": 1024, + "type": "keyword" + }, + "logger": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "rule": { + "properties": { + "author": { + "ignore_above": 1024, + "type": "keyword", + "meta": { + "isArray": "true" + } + }, + "category": { + "ignore_above": 1024, + "type": "keyword" + }, + "description": { + "ignore_above": 1024, + "type": "keyword" + }, + "id": { + "ignore_above": 1024, + "type": "keyword" + }, + "license": { + "ignore_above": 1024, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "reference": { + "ignore_above": 1024, + "type": "keyword" + }, + "ruleset": { + "ignore_above": 1024, + "type": "keyword" + }, + "uuid": { + "ignore_above": 1024, + "type": "keyword" + }, + "version": { + "ignore_above": 1024, + "type": "keyword" } } }, @@ -101,6 +258,7 @@ } }, "saved_objects": { + "type": "nested", "properties": { "rel": { "type": "keyword", @@ -118,8 +276,7 @@ "type": "keyword", "ignore_above": 1024 } - }, - "type": "nested" + } } } } diff --git a/x-pack/plugins/event_log/generated/schemas.ts b/x-pack/plugins/event_log/generated/schemas.ts index 030815ce3c3d..31d8b7201cfc 100644 --- a/x-pack/plugins/event_log/generated/schemas.ts +++ b/x-pack/plugins/event_log/generated/schemas.ts @@ -19,7 +19,7 @@ type DeepPartial = { [P in keyof T]?: T[P] extends Array ? Array> : DeepPartial; }; -export const ECS_VERSION = '1.6.0'; +export const ECS_VERSION = '1.8.0'; // types and config-schema describing the es structures export type IValidatedEvent = TypeOf; @@ -28,27 +28,69 @@ export type IEvent = DeepPartial>; export const EventSchema = schema.maybe( schema.object({ '@timestamp': ecsDate(), - tags: ecsStringMulti(), message: ecsString(), + tags: ecsStringMulti(), ecs: schema.maybe( schema.object({ version: ecsString(), }) ), + error: schema.maybe( + schema.object({ + code: ecsString(), + id: ecsString(), + message: ecsString(), + stack_trace: ecsString(), + type: ecsString(), + }) + ), event: schema.maybe( schema.object({ action: ecsString(), - provider: ecsString(), - start: ecsDate(), + category: ecsStringMulti(), + code: ecsString(), + created: ecsDate(), + dataset: ecsString(), duration: ecsNumber(), end: ecsDate(), + hash: ecsString(), + id: ecsString(), + ingested: ecsDate(), + kind: ecsString(), + module: ecsString(), + original: ecsString(), outcome: ecsString(), + provider: ecsString(), reason: ecsString(), + reference: ecsString(), + risk_score: ecsNumber(), + risk_score_norm: ecsNumber(), + sequence: ecsNumber(), + severity: ecsNumber(), + start: ecsDate(), + timezone: ecsString(), + type: ecsStringMulti(), + url: ecsString(), }) ), - error: schema.maybe( + log: schema.maybe( schema.object({ - message: ecsString(), + level: ecsString(), + logger: ecsString(), + }) + ), + rule: schema.maybe( + schema.object({ + author: ecsStringMulti(), + category: ecsString(), + description: ecsString(), + id: ecsString(), + license: ecsString(), + name: ecsString(), + reference: ecsString(), + ruleset: ecsString(), + uuid: ecsString(), + version: ecsString(), }) ), user: schema.maybe( diff --git a/x-pack/plugins/event_log/scripts/create_schemas.js b/x-pack/plugins/event_log/scripts/create_schemas.js index 98140ef90519..4b91cf6a7362 100755 --- a/x-pack/plugins/event_log/scripts/create_schemas.js +++ b/x-pack/plugins/event_log/scripts/create_schemas.js @@ -27,9 +27,12 @@ function main() { const ecsMappings = readEcsJSONFile(ecsDir, ECS_MAPPINGS_FILE); // add our custom fields - ecsMappings.mappings.properties.kibana = mappings.EcsKibanaExtensionsMappings; + ecsMappings.mappings.properties = { + ...ecsMappings.mappings.properties, + ...mappings.EcsCustomPropertyMappings, + }; - const exportedProperties = mappings.EcsEventLogProperties; + const exportedProperties = mappings.EcsPropertiesToGenerate; const multiValuedProperties = new Set(mappings.EcsEventLogMultiValuedProperties); augmentMappings(ecsMappings.mappings, multiValuedProperties); diff --git a/x-pack/plugins/event_log/scripts/mappings.js b/x-pack/plugins/event_log/scripts/mappings.js index 3d2deda65f48..a7e5f4ae6cb1 100644 --- a/x-pack/plugins/event_log/scripts/mappings.js +++ b/x-pack/plugins/event_log/scripts/mappings.js @@ -5,87 +5,86 @@ * 2.0. */ -exports.EcsKibanaExtensionsMappings = { - properties: { - // kibana server uuid - server_uuid: { - type: 'keyword', - ignore_above: 1024, - }, - // alerting specific fields - alerting: { - properties: { - instance_id: { - type: 'keyword', - ignore_above: 1024, - }, - action_group_id: { - type: 'keyword', - ignore_above: 1024, - }, - action_subgroup: { - type: 'keyword', - ignore_above: 1024, - }, - status: { - type: 'keyword', - ignore_above: 1024, +/** + * These are mappings of custom properties that are not part of ECS. + * Must not interfere with standard ECS fields and field sets. + */ +exports.EcsCustomPropertyMappings = { + kibana: { + properties: { + // kibana server uuid + server_uuid: { + type: 'keyword', + ignore_above: 1024, + }, + // alerting specific fields + alerting: { + properties: { + instance_id: { + type: 'keyword', + ignore_above: 1024, + }, + action_group_id: { + type: 'keyword', + ignore_above: 1024, + }, + action_subgroup: { + type: 'keyword', + ignore_above: 1024, + }, + status: { + type: 'keyword', + ignore_above: 1024, + }, }, }, - }, - // array of saved object references, for "linking" via search - saved_objects: { - type: 'nested', - properties: { - // relation; currently only supports "primary" or not set - rel: { - type: 'keyword', - ignore_above: 1024, - }, - // relevant kibana space - namespace: { - type: 'keyword', - ignore_above: 1024, - }, - id: { - type: 'keyword', - ignore_above: 1024, - }, - type: { - type: 'keyword', - ignore_above: 1024, + // array of saved object references, for "linking" via search + saved_objects: { + type: 'nested', + properties: { + // relation; currently only supports "primary" or not set + rel: { + type: 'keyword', + ignore_above: 1024, + }, + // relevant kibana space + namespace: { + type: 'keyword', + ignore_above: 1024, + }, + id: { + type: 'keyword', + ignore_above: 1024, + }, + type: { + type: 'keyword', + ignore_above: 1024, + }, }, }, }, }, }; -// ECS and Kibana ECS extension properties to generate -exports.EcsEventLogProperties = [ +/** + * These properties will be added to the generated event schema. + * Here you can specify single fields (log.level) and whole field sets (event). + */ +exports.EcsPropertiesToGenerate = [ '@timestamp', - 'tags', 'message', - 'ecs.version', - 'event.action', - 'event.provider', - 'event.start', - 'event.duration', - 'event.end', - 'event.outcome', // optional, but one of failure, success, unknown - 'event.reason', - 'error.message', + 'tags', + 'ecs', + 'error', + 'event', + 'log.level', + 'log.logger', + 'rule', 'user.name', - 'kibana.server_uuid', - 'kibana.alerting.instance_id', - 'kibana.alerting.action_group_id', - 'kibana.alerting.action_subgroup', - 'kibana.alerting.status', - 'kibana.saved_objects.rel', - 'kibana.saved_objects.namespace', - 'kibana.saved_objects.id', - 'kibana.saved_objects.name', - 'kibana.saved_objects.type', + 'kibana', ]; -// properties that can have multiple values (array vs single value) -exports.EcsEventLogMultiValuedProperties = ['tags']; +/** + * These properties can have multiple values (are arrays in the generated event schema). + */ +exports.EcsEventLogMultiValuedProperties = ['tags', 'event.category', 'event.type', 'rule.author']; diff --git a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts index df81de6d8c3d..f9f518091847 100644 --- a/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts +++ b/x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts @@ -5,6 +5,7 @@ * 2.0. */ +import _ from 'lodash'; import uuid from 'uuid'; import expect from '@kbn/expect/expect.js'; import { IEvent } from '../../../../plugins/event_log/server'; @@ -77,30 +78,108 @@ export default function ({ getService }: FtrProviderContext) { await registerProviderActions('provider1', ['action1', 'action2']); } - const providerActions = await getProviderActions('provider1'); + const providerActions = await getRegisteredProviderActions('provider1'); expect(providerActions.body.actions).to.be.eql(['action1', 'action2']); }); - it('should allow write an event to index document if indexing entries is enabled', async () => { - const initResult = await isProviderActionRegistered('provider4', 'action1'); - - if (!initResult.body.isProviderActionRegistered) { - await registerProviderActions('provider4', ['action1', 'action2']); - } - - const eventId = uuid.v4(); + it('should allow to log an event and then find it by saved object', async () => { + const { provider, action } = await getTestProviderAction(); + const savedObject = getTestSavedObject(); const event: IEvent = { - event: { action: 'action1', provider: 'provider4' }, - kibana: { saved_objects: [{ rel: 'primary', type: 'event_log_test', id: eventId }] }, + event: { provider, action }, + kibana: { saved_objects: [savedObject] }, }; - await logTestEvent(eventId, event); - await retry.try(async () => { - const uri = `/api/event_log/event_log_test/${eventId}/_find`; - log.debug(`calling ${uri}`); - const result = await supertest.get(uri).set('kbn-xsrf', 'foo').expect(200); - expect(result.body.data.length).to.be.eql(1); - }); + const indexedEvent = await logAndWaitUntilIndexed(event, savedObject.type, savedObject.id); + + expect(indexedEvent.event.provider).to.be.eql(event.event?.provider); + expect(indexedEvent.event.action).to.be.eql(event.event?.action); + }); + + it('should respect event schema - properly index and preserve all the properties of an event', async () => { + const { provider, action } = await getTestProviderAction(); + const savedObject = getTestSavedObject(); + const event: IEvent = { + '@timestamp': '2042-03-25T11:53:24.911Z', + message: 'some message', + tags: ['some', 'tags'], + event: { + provider, + action, + category: ['some', 'categories'], + code: '4242', + created: '2042-03-25T11:53:24.911Z', + dataset: `${provider}.dataset`, + hash: '123456789012345678901234567890ABCD', + id: '98506718-03ec-4d0a-a7bd-b8459a60a82d', + ingested: '2042-03-25T11:53:24.911Z', + kind: 'event', + module: `${provider}.module`, + original: 'Sep 19 08:26:10 host CEF:0|Security| worm successfully stopped', + outcome: 'success', + reason: 'Terminated an unexpected process', + reference: 'https://system.example.com/event/#0001234', + risk_score: 987.65, + risk_score_norm: 42.5, + sequence: 1234567890, + severity: 20, + timezone: 'Europe/Amsterdam', + type: ['change', 'info'], + url: 'https://mysystem.example.com/alert/5271dedb-f5b0-4218-87f0-4ac4870a38fe', + }, + error: { + code: '42000', + id: 'f53be013-8a8c-4d39-b0f0-2781bb088a33', + message: 'Unexpected error', + stack_trace: `Error: Unexpected error + at Context. (x-pack/test/plugin_api_integration/test_suites/event_log/service_api_integration.ts:160:13) + at Object.apply (node_modules/@kbn/test/src/functional_test_runner/lib/mocha/wrap_function.js:73:16) + at Object.apply (node_modules/@kbn/test/src/functional_test_runner/lib/mocha/wrap_function.js:73:16)`, + type: 'Error', + }, + log: { + level: 'warning', + logger: `${provider}.child-logger`, + }, + rule: { + author: ['Elastic', 'Security'], + category: 'Attempted Information Leak', + description: 'Block requests to public DNS over HTTPS / TLS protocols', + id: '101', + license: 'Apache 2.0', + name: 'BLOCK_DNS_over_TLS', + reference: 'https://en.wikipedia.org/wiki/DNS_over_TLS', + ruleset: 'Standard_Protocol_Filters', + uuid: '1fd3e1ad-1376-406f-ac63-df97db5d2fae', + version: '1.1', + }, + user: { + name: 'elastic', + }, + kibana: { + saved_objects: [savedObject], + alerting: { + instance_id: 'alert instance id', + action_group_id: 'alert action group', + action_subgroup: 'alert action subgroup', + status: 'overall alert status, after alert execution', + }, + }, + }; + + const indexedEvent = await logAndWaitUntilIndexed(event, savedObject.type, savedObject.id); + + // Omit properties which are set by the event logger + // NOTE: event.* properties are set by the `/api/log_event_fixture/${savedObjectId}/_log` route handler + const propertiesToCheck = _.omit(indexedEvent, [ + 'ecs', + 'event.start', + 'event.end', + 'event.duration', + 'kibana.server_uuid', + ]); + + expect(propertiesToCheck).to.be.eql(event); }); }); @@ -120,7 +199,7 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); } - async function getProviderActions(provider: string) { + async function getRegisteredProviderActions(provider: string) { log.debug(`getProviderActions ${provider}`); return await supertest .get(`/api/log_event_fixture/${provider}/getProviderActions`) @@ -152,12 +231,53 @@ export default function ({ getService }: FtrProviderContext) { .expect(200); } - async function logTestEvent(id: string, event: IEvent) { - log.debug(`Logging Event for Saved Object ${id}`); + async function getTestProviderAction() { + const provider = `provider-${uuid.v4()}`; + const action = `action-${uuid.v4()}`; + + const response = await isProviderActionRegistered(provider, action); + if (!response.body.isProviderActionRegistered) { + await registerProviderActions(provider, [action]); + } + + return { provider, action }; + } + + function getTestSavedObject() { + return { type: 'event_log_test', id: uuid.v4(), rel: 'primary' }; + } + + async function logEvent(event: IEvent, savedObjectId: string) { + log.debug(`Logging Event for Saved Object ${savedObjectId}`); return await supertest - .post(`/api/log_event_fixture/${id}/_log`) + .post(`/api/log_event_fixture/${savedObjectId}/_log`) .set('kbn-xsrf', 'foo') .send(event) .expect(200); } + + async function fetchEvents(savedObjectType: string, savedObjectId: string) { + log.debug(`Fetching events of Saved Object ${savedObjectId}`); + return await supertest + .get(`/api/event_log/${savedObjectType}/${savedObjectId}/_find`) + .set('kbn-xsrf', 'foo') + .expect(200); + } + + // NOTE: It supports indexing only 1 event per test. + async function logAndWaitUntilIndexed( + event: IEvent, + savedObjectType: string, + savedObjectId: string + ) { + await logEvent(event, savedObjectId); + + const response = await retry.tryForTime(30000, async () => { + const res = await fetchEvents(savedObjectType, savedObjectId); + expect(res.body.data.length).to.be.eql(1); + return res; + }); + + return response.body.data[0]; + } }