From 44f7a99e6f3d53998227614cf10c33e4a4889bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20St=C3=BCrmer?= Date: Mon, 19 Jul 2021 19:56:06 +0200 Subject: [PATCH] [Observability RAC] Remove indexing of rule evaluation documents (#104970) --- x-pack/plugins/rule_registry/server/index.ts | 2 +- .../create_rule_data_client_mock.ts | 31 +- .../server/rule_data_client/types.ts | 1 + .../utils/create_lifecycle_executor.test.ts | 374 ++++++++++++++++++ .../server/utils/create_lifecycle_executor.ts | 19 +- .../utils/create_lifecycle_rule_type.test.ts | 48 +-- .../server/utils/get_rule_executor_data.ts | 15 - 7 files changed, 400 insertions(+), 90 deletions(-) create mode 100644 x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts diff --git a/x-pack/plugins/rule_registry/server/index.ts b/x-pack/plugins/rule_registry/server/index.ts index 6b0765e71cba..f8d9dec3ea83 100644 --- a/x-pack/plugins/rule_registry/server/index.ts +++ b/x-pack/plugins/rule_registry/server/index.ts @@ -13,7 +13,7 @@ export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract } export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types'; export { RuleDataClient } from './rule_data_client'; export { IRuleDataClient } from './rule_data_client/types'; -export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data'; +export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data'; export { createLifecycleRuleTypeFactory, LifecycleAlertService, diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts b/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts index 59f740e0afb7..24b06439fe57 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts @@ -4,24 +4,31 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { Assign } from '@kbn/utility-types'; +import { PublicContract } from '@kbn/utility-types'; import type { RuleDataClient } from '.'; import { RuleDataReader, RuleDataWriter } from './types'; type MockInstances> = { [K in keyof T]: T[K] extends (...args: infer TArgs) => infer TReturn - ? jest.MockInstance + ? jest.MockInstance & T[K] : never; }; -export function createRuleDataClientMock() { +type RuleDataClientMock = jest.Mocked< + Omit, 'getWriter' | 'getReader'> +> & { + getWriter: (...args: Parameters) => MockInstances; + getReader: (...args: Parameters) => MockInstances; +}; + +export function createRuleDataClientMock(): RuleDataClientMock { const bulk = jest.fn(); const search = jest.fn(); const getDynamicIndexPattern = jest.fn(); - return ({ - createOrUpdateWriteTarget: jest.fn(({ namespace }) => Promise.resolve()), - getReader: jest.fn(() => ({ + return { + createWriteTargetIfNeeded: jest.fn(({}) => Promise.resolve()), + getReader: jest.fn((_options?: { namespace?: string }) => ({ getDynamicIndexPattern, search, })), @@ -29,15 +36,5 @@ export function createRuleDataClientMock() { bulk, })), isWriteEnabled: jest.fn(() => true), - } as unknown) as Assign< - RuleDataClient & Omit, 'options' | 'getClusterClient'>, - { - getWriter: ( - ...args: Parameters - ) => MockInstances; - getReader: ( - ...args: Parameters - ) => MockInstances; - } - >; + }; } diff --git a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts index 54e9a1b3c9a6..92ba5c7060eb 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_client/types.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_client/types.ts @@ -35,6 +35,7 @@ export interface RuleDataWriter { export interface IRuleDataClient { getReader(options?: { namespace?: string }): RuleDataReader; getWriter(options?: { namespace?: string }): RuleDataWriter; + isWriteEnabled(): boolean; createWriteTargetIfNeeded(options: { namespace?: string }): Promise; } diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts new file mode 100644 index 000000000000..a036f4273999 --- /dev/null +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts @@ -0,0 +1,374 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { loggerMock } from '@kbn/logging/target/mocks'; +import { + elasticsearchServiceMock, + savedObjectsClientMock, +} from '../../../../../src/core/server/mocks'; +import { + AlertExecutorOptions, + AlertInstanceContext, + AlertInstanceState, + AlertTypeParams, + AlertTypeState, +} from '../../../alerting/server'; +import { alertsMock } from '../../../alerting/server/mocks'; +import { + ALERT_ID, + ALERT_STATUS, + EVENT_ACTION, + EVENT_KIND, +} from '../../common/technical_rule_data_field_names'; +import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock'; +import { createLifecycleExecutor } from './create_lifecycle_executor'; + +describe('createLifecycleExecutor', () => { + it('wraps and unwraps the original executor state', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async (options) => { + expect(options.state).toEqual(initialRuleState); + + const nextRuleState: TestRuleState = { + aRuleStateKey: 'NEXT_RULE_STATE_VALUE', + }; + + return nextRuleState; + }); + + const newRuleState = await executor( + createDefaultAlertExecutorOptions({ + params: {}, + state: { wrapped: initialRuleState, trackedAlerts: {} }, + }) + ); + + expect(newRuleState).toEqual({ + wrapped: { + aRuleStateKey: 'NEXT_RULE_STATE_VALUE', + }, + trackedAlerts: {}, + }); + }); + + it('writes initial documents for newly firing alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + services.alertWithLifecycle({ + id: 'TEST_ALERT_0', + fields: {}, + }); + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + params: {}, + state: { wrapped: initialRuleState, trackedAlerts: {} }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: [ + // alert documents + { index: { _id: expect.any(String) } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'open', + [EVENT_KIND]: 'signal', + }), + { index: { _id: expect.any(String) } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'open', + [EVENT_KIND]: 'signal', + }), + ], + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); + + it('overwrites existing documents for repeatedly firing alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + ruleDataClientMock.getReader().search.mockResolvedValue({ + hits: { + hits: [ + { + fields: { + [ALERT_ID]: 'TEST_ALERT_0', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + { + fields: { + [ALERT_ID]: 'TEST_ALERT_1', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + ], + }, + } as any); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + services.alertWithLifecycle({ + id: 'TEST_ALERT_0', + fields: {}, + }); + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + alertId: 'TEST_ALERT_0', + params: {}, + state: { + wrapped: initialRuleState, + trackedAlerts: { + TEST_ALERT_0: { + alertId: 'TEST_ALERT_0', + alertUuid: 'TEST_ALERT_0_UUID', + started: '2020-01-01T12:00:00.000Z', + }, + TEST_ALERT_1: { + alertId: 'TEST_ALERT_1', + alertUuid: 'TEST_ALERT_1_UUID', + started: '2020-01-02T12:00:00.000Z', + }, + }, + }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: [ + // alert document + { index: { _id: 'TEST_ALERT_0_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + { index: { _id: 'TEST_ALERT_1_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + ], + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); + + it('updates existing documents for recovered alerts', async () => { + const logger = loggerMock.create(); + const ruleDataClientMock = createRuleDataClientMock(); + ruleDataClientMock.getReader().search.mockResolvedValue({ + hits: { + hits: [ + { + fields: { + '@timestamp': '', + [ALERT_ID]: 'TEST_ALERT_0', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc + }, + }, + { + fields: { + '@timestamp': '', + [ALERT_ID]: 'TEST_ALERT_1', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc + }, + }, + ], + }, + } as any); + const executor = createLifecycleExecutor( + logger, + ruleDataClientMock + )<{}, TestRuleState, never, never, never>(async ({ services, state }) => { + // TEST_ALERT_0 has recovered + services.alertWithLifecycle({ + id: 'TEST_ALERT_1', + fields: {}, + }); + + return state; + }); + + await executor( + createDefaultAlertExecutorOptions({ + alertId: 'TEST_ALERT_0', + params: {}, + state: { + wrapped: initialRuleState, + trackedAlerts: { + TEST_ALERT_0: { + alertId: 'TEST_ALERT_0', + alertUuid: 'TEST_ALERT_0_UUID', + started: '2020-01-01T12:00:00.000Z', + }, + TEST_ALERT_1: { + alertId: 'TEST_ALERT_1', + alertUuid: 'TEST_ALERT_1_UUID', + started: '2020-01-02T12:00:00.000Z', + }, + }, + }, + }) + ); + + expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // alert document + { index: { _id: 'TEST_ALERT_0_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_0', + [ALERT_STATUS]: 'closed', + labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, + [EVENT_ACTION]: 'close', + [EVENT_KIND]: 'signal', + }), + { index: { _id: 'TEST_ALERT_1_UUID' } }, + expect.objectContaining({ + [ALERT_ID]: 'TEST_ALERT_1', + [ALERT_STATUS]: 'open', + [EVENT_ACTION]: 'active', + [EVENT_KIND]: 'signal', + }), + ]), + }) + ); + expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.arrayContaining([ + // evaluation documents + { index: {} }, + expect.objectContaining({ + [EVENT_KIND]: 'event', + }), + ]), + }) + ); + }); +}); + +type TestRuleState = Record & { + aRuleStateKey: string; +}; + +const initialRuleState: TestRuleState = { + aRuleStateKey: 'INITIAL_RULE_STATE_VALUE', +}; + +const createDefaultAlertExecutorOptions = < + Params extends AlertTypeParams = never, + State extends AlertTypeState = never, + InstanceState extends AlertInstanceState = {}, + InstanceContext extends AlertInstanceContext = {}, + ActionGroupIds extends string = '' +>({ + alertId = 'ALERT_ID', + ruleName = 'RULE_NAME', + params, + state, + createdAt = new Date(), + startedAt = new Date(), + updatedAt = new Date(), +}: { + alertId?: string; + ruleName?: string; + params: Params; + state: State; + createdAt?: Date; + startedAt?: Date; + updatedAt?: Date; +}): AlertExecutorOptions => ({ + alertId, + createdBy: 'CREATED_BY', + startedAt, + name: ruleName, + rule: { + updatedBy: null, + tags: [], + name: ruleName, + createdBy: null, + actions: [], + enabled: true, + consumer: 'CONSUMER', + producer: 'PRODUCER', + schedule: { interval: '1m' }, + throttle: null, + createdAt, + updatedAt, + notifyWhen: null, + ruleTypeId: 'RULE_TYPE_ID', + ruleTypeName: 'RULE_TYPE_NAME', + }, + tags: [], + params, + spaceId: 'SPACE_ID', + services: { + alertInstanceFactory: alertsMock.createAlertServices() + .alertInstanceFactory, + savedObjectsClient: savedObjectsClientMock.create(), + scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(), + }, + state, + updatedBy: null, + previousStartedAt: null, + namespace: undefined, +}); diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts index 8df343fb16d4..2a18f28710d0 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts @@ -5,7 +5,8 @@ * 2.0. */ -import { Logger } from '@kbn/logging'; +import type { Logger } from '@kbn/logging'; +import type { PublicContract } from '@kbn/utility-types'; import { getOrElse } from 'fp-ts/lib/Either'; import * as rt from 'io-ts'; import { Mutable } from 'utility-types'; @@ -98,7 +99,10 @@ export type WrappedLifecycleRuleState = AlertTypeS trackedAlerts: Record; }; -export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleDataClient) => < +export const createLifecycleExecutor = ( + logger: Logger, + ruleDataClient: PublicContract +) => < Params extends AlertTypeParams = never, State extends AlertTypeState = never, InstanceState extends AlertInstanceState = never, @@ -242,7 +246,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData ...alertData, ...ruleExecutorData, [TIMESTAMP]: timestamp, - [EVENT_KIND]: 'event', + [EVENT_KIND]: 'signal', [OWNER]: rule.consumer, [ALERT_ID]: alertId, }; @@ -311,14 +315,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData if (ruleDataClient.isWriteEnabled()) { await ruleDataClient.getWriter().bulk({ - body: eventsToIndex - .flatMap((event) => [{ index: {} }, event]) - .concat( - Array.from(alertEvents.values()).flatMap((event) => [ - { index: { _id: event[ALERT_UUID]! } }, - event, - ]) - ), + body: eventsToIndex.flatMap((event) => [{ index: { _id: event[ALERT_UUID]! } }, event]), }); } } diff --git a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts index e26e5b00435f..11bb48a7440a 100644 --- a/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts +++ b/x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts @@ -173,7 +173,7 @@ describe('createLifecycleRuleTypeFactory', () => { const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event'); const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal'); - expect(evaluationDocuments.length).toBe(2); + expect(evaluationDocuments.length).toBe(0); expect(alertDocuments.length).toBe(2); expect( @@ -188,50 +188,6 @@ describe('createLifecycleRuleTypeFactory', () => { expect(documents.map((doc) => omit(doc, 'kibana.rac.alert.uuid'))).toMatchInlineSnapshot(` Array [ - Object { - "@timestamp": "2021-06-16T09:01:00.000Z", - "event.action": "open", - "event.kind": "event", - "kibana.rac.alert.duration.us": 0, - "kibana.rac.alert.id": "opbeans-java", - "kibana.rac.alert.owner": "consumer", - "kibana.rac.alert.producer": "producer", - "kibana.rac.alert.start": "2021-06-16T09:01:00.000Z", - "kibana.rac.alert.status": "open", - "kibana.space_ids": Array [ - "spaceId", - ], - "rule.category": "ruleTypeName", - "rule.id": "ruleTypeId", - "rule.name": "name", - "rule.uuid": "alertId", - "service.name": "opbeans-java", - "tags": Array [ - "tags", - ], - }, - Object { - "@timestamp": "2021-06-16T09:01:00.000Z", - "event.action": "open", - "event.kind": "event", - "kibana.rac.alert.duration.us": 0, - "kibana.rac.alert.id": "opbeans-node", - "kibana.rac.alert.owner": "consumer", - "kibana.rac.alert.producer": "producer", - "kibana.rac.alert.start": "2021-06-16T09:01:00.000Z", - "kibana.rac.alert.status": "open", - "kibana.space_ids": Array [ - "spaceId", - ], - "rule.category": "ruleTypeName", - "rule.id": "ruleTypeId", - "rule.name": "name", - "rule.uuid": "alertId", - "service.name": "opbeans-node", - "tags": Array [ - "tags", - ], - }, Object { "@timestamp": "2021-06-16T09:01:00.000Z", "event.action": "open", @@ -324,7 +280,7 @@ describe('createLifecycleRuleTypeFactory', () => { const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event'); const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal'); - expect(evaluationDocuments.length).toBe(2); + expect(evaluationDocuments.length).toBe(0); expect(alertDocuments.length).toBe(2); expect( diff --git a/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts index 7cb02428322a..144c0dafa378 100644 --- a/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts +++ b/x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts @@ -14,7 +14,6 @@ import { RULE_UUID, TAGS, } from '../../common/technical_rule_data_field_names'; -import { AlertTypeExecutor, AlertTypeWithExecutor } from '../types'; export interface RuleExecutorData { [RULE_CATEGORY]: string; @@ -25,20 +24,6 @@ export interface RuleExecutorData { [TAGS]: string[]; } -export function getRuleExecutorData( - type: AlertTypeWithExecutor, - options: Parameters[0] -) { - return { - [RULE_ID]: type.id, - [RULE_UUID]: options.alertId, - [RULE_CATEGORY]: type.name, - [RULE_NAME]: options.name, - [TAGS]: options.tags, - [PRODUCER]: type.producer, - }; -} - export function getRuleData(options: AlertExecutorOptions) { return { [RULE_ID]: options.rule.ruleTypeId,