maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Observability RAC] Remove indexing of rule evaluation documents (#104970)

Browse source
This commit is contained in:
Felix Stürmer 2021-07-19 19:56:06 +02:00 committed by GitHub
parent 595056060a
commit 44f7a99e6f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 400 additions and 90 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/plugins/rule_registry/server/index.ts
View file

@ -13,7 +13,7 @@ export type { RuleRegistryPluginSetupContract, RuleRegistryPluginStartContract }
export type { RacRequestHandlerContext, RacApiRequestHandlerContext } from './types';
export { RuleDataClient } from './rule_data_client';
export { IRuleDataClient } from './rule_data_client/types';
export { getRuleExecutorData, RuleExecutorData } from './utils/get_rule_executor_data';
export { getRuleData, RuleExecutorData } from './utils/get_rule_executor_data';
export {
createLifecycleRuleTypeFactory,
LifecycleAlertService,

31
x-pack/plugins/rule_registry/server/rule_data_client/create_rule_data_client_mock.ts
View file

@ -4,24 +4,31 @@
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { Assign } from '@kbn/utility-types';
import { PublicContract } from '@kbn/utility-types';
import type { RuleDataClient } from '.';
import { RuleDataReader, RuleDataWriter } from './types';
type MockInstances<T extends Record<string, any>> = {
[K in keyof T]: T[K] extends (...args: infer TArgs) => infer TReturn
? jest.MockInstance<TReturn, TArgs>
? jest.MockInstance<TReturn, TArgs> & T[K]
: never;
};
export function createRuleDataClientMock() {
type RuleDataClientMock = jest.Mocked<
Omit<PublicContract<RuleDataClient>, 'getWriter' | 'getReader'>
> & {
getWriter: (...args: Parameters<RuleDataClient['getWriter']>) => MockInstances<RuleDataWriter>;
getReader: (...args: Parameters<RuleDataClient['getReader']>) => MockInstances<RuleDataReader>;
};
export function createRuleDataClientMock(): RuleDataClientMock {
const bulk = jest.fn();
const search = jest.fn();
const getDynamicIndexPattern = jest.fn();
return ({
createOrUpdateWriteTarget: jest.fn(({ namespace }) => Promise.resolve()),
getReader: jest.fn(() => ({
return {
createWriteTargetIfNeeded: jest.fn(({}) => Promise.resolve()),
getReader: jest.fn((_options?: { namespace?: string }) => ({
getDynamicIndexPattern,
search,
})),
@ -29,15 +36,5 @@ export function createRuleDataClientMock() {
bulk,
})),
isWriteEnabled: jest.fn(() => true),
} as unknown) as Assign<
RuleDataClient & Omit<MockInstances<RuleDataClient>, 'options' | 'getClusterClient'>,
{
getWriter: (
...args: Parameters<RuleDataClient['getWriter']>
) => MockInstances<RuleDataWriter>;
getReader: (
...args: Parameters<RuleDataClient['getReader']>
) => MockInstances<RuleDataReader>;
}
>;
};
}

1
x-pack/plugins/rule_registry/server/rule_data_client/types.ts
View file

@ -35,6 +35,7 @@ export interface RuleDataWriter {
export interface IRuleDataClient {
getReader(options?: { namespace?: string }): RuleDataReader;
getWriter(options?: { namespace?: string }): RuleDataWriter;
isWriteEnabled(): boolean;
createWriteTargetIfNeeded(options: { namespace?: string }): Promise<void>;
}

374
x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.test.ts Normal file
View file

@ -0,0 +1,374 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { loggerMock } from '@kbn/logging/target/mocks';
import {
elasticsearchServiceMock,
savedObjectsClientMock,
} from '../../../../../src/core/server/mocks';
import {
AlertExecutorOptions,
AlertInstanceContext,
AlertInstanceState,
AlertTypeParams,
AlertTypeState,
} from '../../../alerting/server';
import { alertsMock } from '../../../alerting/server/mocks';
import {
ALERT_ID,
ALERT_STATUS,
EVENT_ACTION,
EVENT_KIND,
} from '../../common/technical_rule_data_field_names';
import { createRuleDataClientMock } from '../rule_data_client/create_rule_data_client_mock';
import { createLifecycleExecutor } from './create_lifecycle_executor';
describe('createLifecycleExecutor', () => {
it('wraps and unwraps the original executor state', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
const executor = createLifecycleExecutor(
logger,
ruleDataClientMock
)<{}, TestRuleState, never, never, never>(async (options) => {
expect(options.state).toEqual(initialRuleState);
const nextRuleState: TestRuleState = {
aRuleStateKey: 'NEXT_RULE_STATE_VALUE',
};
return nextRuleState;
});
const newRuleState = await executor(
createDefaultAlertExecutorOptions({
params: {},
state: { wrapped: initialRuleState, trackedAlerts: {} },
})
);
expect(newRuleState).toEqual({
wrapped: {
aRuleStateKey: 'NEXT_RULE_STATE_VALUE',
},
trackedAlerts: {},
});
});
it('writes initial documents for newly firing alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
const executor = createLifecycleExecutor(
logger,
ruleDataClientMock
)<{}, TestRuleState, never, never, never>(async ({ services, state }) => {
services.alertWithLifecycle({
id: 'TEST_ALERT_0',
fields: {},
});
services.alertWithLifecycle({
id: 'TEST_ALERT_1',
fields: {},
});
return state;
});
await executor(
createDefaultAlertExecutorOptions({
params: {},
state: { wrapped: initialRuleState, trackedAlerts: {} },
})
);
expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith(
expect.objectContaining({
body: [
// alert documents
{ index: { _id: expect.any(String) } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_0',
[ALERT_STATUS]: 'open',
[EVENT_ACTION]: 'open',
[EVENT_KIND]: 'signal',
}),
{ index: { _id: expect.any(String) } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_1',
[ALERT_STATUS]: 'open',
[EVENT_ACTION]: 'open',
[EVENT_KIND]: 'signal',
}),
],
})
);
expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
// evaluation documents
{ index: {} },
expect.objectContaining({
[EVENT_KIND]: 'event',
}),
]),
})
);
});
it('overwrites existing documents for repeatedly firing alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
hits: {
hits: [
{
fields: {
[ALERT_ID]: 'TEST_ALERT_0',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
},
},
{
fields: {
[ALERT_ID]: 'TEST_ALERT_1',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
},
},
],
},
} as any);
const executor = createLifecycleExecutor(
logger,
ruleDataClientMock
)<{}, TestRuleState, never, never, never>(async ({ services, state }) => {
services.alertWithLifecycle({
id: 'TEST_ALERT_0',
fields: {},
});
services.alertWithLifecycle({
id: 'TEST_ALERT_1',
fields: {},
});
return state;
});
await executor(
createDefaultAlertExecutorOptions({
alertId: 'TEST_ALERT_0',
params: {},
state: {
wrapped: initialRuleState,
trackedAlerts: {
TEST_ALERT_0: {
alertId: 'TEST_ALERT_0',
alertUuid: 'TEST_ALERT_0_UUID',
started: '2020-01-01T12:00:00.000Z',
},
TEST_ALERT_1: {
alertId: 'TEST_ALERT_1',
alertUuid: 'TEST_ALERT_1_UUID',
started: '2020-01-02T12:00:00.000Z',
},
},
},
})
);
expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith(
expect.objectContaining({
body: [
// alert document
{ index: { _id: 'TEST_ALERT_0_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_0',
[ALERT_STATUS]: 'open',
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
}),
{ index: { _id: 'TEST_ALERT_1_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_1',
[ALERT_STATUS]: 'open',
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
}),
],
})
);
expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
// evaluation documents
{ index: {} },
expect.objectContaining({
[EVENT_KIND]: 'event',
}),
]),
})
);
});
it('updates existing documents for recovered alerts', async () => {
const logger = loggerMock.create();
const ruleDataClientMock = createRuleDataClientMock();
ruleDataClientMock.getReader().search.mockResolvedValue({
hits: {
hits: [
{
fields: {
'@timestamp': '',
[ALERT_ID]: 'TEST_ALERT_0',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must show up in the written doc
},
},
{
fields: {
'@timestamp': '',
[ALERT_ID]: 'TEST_ALERT_1',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' }, // this must not show up in the written doc
},
},
],
},
} as any);
const executor = createLifecycleExecutor(
logger,
ruleDataClientMock
)<{}, TestRuleState, never, never, never>(async ({ services, state }) => {
// TEST_ALERT_0 has recovered
services.alertWithLifecycle({
id: 'TEST_ALERT_1',
fields: {},
});
return state;
});
await executor(
createDefaultAlertExecutorOptions({
alertId: 'TEST_ALERT_0',
params: {},
state: {
wrapped: initialRuleState,
trackedAlerts: {
TEST_ALERT_0: {
alertId: 'TEST_ALERT_0',
alertUuid: 'TEST_ALERT_0_UUID',
started: '2020-01-01T12:00:00.000Z',
},
TEST_ALERT_1: {
alertId: 'TEST_ALERT_1',
alertUuid: 'TEST_ALERT_1_UUID',
started: '2020-01-02T12:00:00.000Z',
},
},
},
})
);
expect(ruleDataClientMock.getWriter().bulk).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
// alert document
{ index: { _id: 'TEST_ALERT_0_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_0',
[ALERT_STATUS]: 'closed',
labels: { LABEL_0_KEY: 'LABEL_0_VALUE' },
[EVENT_ACTION]: 'close',
[EVENT_KIND]: 'signal',
}),
{ index: { _id: 'TEST_ALERT_1_UUID' } },
expect.objectContaining({
[ALERT_ID]: 'TEST_ALERT_1',
[ALERT_STATUS]: 'open',
[EVENT_ACTION]: 'active',
[EVENT_KIND]: 'signal',
}),
]),
})
);
expect(ruleDataClientMock.getWriter().bulk).not.toHaveBeenCalledWith(
expect.objectContaining({
body: expect.arrayContaining([
// evaluation documents
{ index: {} },
expect.objectContaining({
[EVENT_KIND]: 'event',
}),
]),
})
);
});
});
type TestRuleState = Record<string, unknown> & {
aRuleStateKey: string;
};
const initialRuleState: TestRuleState = {
aRuleStateKey: 'INITIAL_RULE_STATE_VALUE',
};
const createDefaultAlertExecutorOptions = <
Params extends AlertTypeParams = never,
State extends AlertTypeState = never,
InstanceState extends AlertInstanceState = {},
InstanceContext extends AlertInstanceContext = {},
ActionGroupIds extends string = ''
>({
alertId = 'ALERT_ID',
ruleName = 'RULE_NAME',
params,
state,
createdAt = new Date(),
startedAt = new Date(),
updatedAt = new Date(),
}: {
alertId?: string;
ruleName?: string;
params: Params;
state: State;
createdAt?: Date;
startedAt?: Date;
updatedAt?: Date;
}): AlertExecutorOptions<Params, State, InstanceState, InstanceContext, ActionGroupIds> => ({
alertId,
createdBy: 'CREATED_BY',
startedAt,
name: ruleName,
rule: {
updatedBy: null,
tags: [],
name: ruleName,
createdBy: null,
actions: [],
enabled: true,
consumer: 'CONSUMER',
producer: 'PRODUCER',
schedule: { interval: '1m' },
throttle: null,
createdAt,
updatedAt,
notifyWhen: null,
ruleTypeId: 'RULE_TYPE_ID',
ruleTypeName: 'RULE_TYPE_NAME',
},
tags: [],
params,
spaceId: 'SPACE_ID',
services: {
alertInstanceFactory: alertsMock.createAlertServices<InstanceState, InstanceContext>()
.alertInstanceFactory,
savedObjectsClient: savedObjectsClientMock.create(),
scopedClusterClient: elasticsearchServiceMock.createScopedClusterClient(),
},
state,
updatedBy: null,
previousStartedAt: null,
namespace: undefined,
});

19
x-pack/plugins/rule_registry/server/utils/create_lifecycle_executor.ts
View file

@ -5,7 +5,8 @@
* 2.0.
*/
import { Logger } from '@kbn/logging';
import type { Logger } from '@kbn/logging';
import type { PublicContract } from '@kbn/utility-types';
import { getOrElse } from 'fp-ts/lib/Either';
import * as rt from 'io-ts';
import { Mutable } from 'utility-types';
@ -98,7 +99,10 @@ export type WrappedLifecycleRuleState<State extends AlertTypeState> = AlertTypeS
trackedAlerts: Record<string, TrackedLifecycleAlertState>;
};
export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleDataClient) => <
export const createLifecycleExecutor = (
logger: Logger,
ruleDataClient: PublicContract<RuleDataClient>
) => <
Params extends AlertTypeParams = never,
State extends AlertTypeState = never,
InstanceState extends AlertInstanceState = never,
@ -242,7 +246,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData
...alertData,
...ruleExecutorData,
[TIMESTAMP]: timestamp,
[EVENT_KIND]: 'event',
[EVENT_KIND]: 'signal',
[OWNER]: rule.consumer,
[ALERT_ID]: alertId,
};
@ -311,14 +315,7 @@ export const createLifecycleExecutor = (logger: Logger, ruleDataClient: RuleData
if (ruleDataClient.isWriteEnabled()) {
await ruleDataClient.getWriter().bulk({
body: eventsToIndex
.flatMap((event) => [{ index: {} }, event])
.concat(
Array.from(alertEvents.values()).flatMap((event) => [
{ index: { _id: event[ALERT_UUID]! } },
event,
])
),
body: eventsToIndex.flatMap((event) => [{ index: { _id: event[ALERT_UUID]! } }, event]),
});
}
}

48
x-pack/plugins/rule_registry/server/utils/create_lifecycle_rule_type.test.ts
View file

@ -173,7 +173,7 @@ describe('createLifecycleRuleTypeFactory', () => {
const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event');
const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal');
expect(evaluationDocuments.length).toBe(2);
expect(evaluationDocuments.length).toBe(0);
expect(alertDocuments.length).toBe(2);
expect(
@ -188,50 +188,6 @@ describe('createLifecycleRuleTypeFactory', () => {
expect(documents.map((doc) => omit(doc, 'kibana.rac.alert.uuid'))).toMatchInlineSnapshot(`
Array [
Object {
"@timestamp": "2021-06-16T09:01:00.000Z",
"event.action": "open",
"event.kind": "event",
"kibana.rac.alert.duration.us": 0,
"kibana.rac.alert.id": "opbeans-java",
"kibana.rac.alert.owner": "consumer",
"kibana.rac.alert.producer": "producer",
"kibana.rac.alert.start": "2021-06-16T09:01:00.000Z",
"kibana.rac.alert.status": "open",
"kibana.space_ids": Array [
"spaceId",
],
"rule.category": "ruleTypeName",
"rule.id": "ruleTypeId",
"rule.name": "name",
"rule.uuid": "alertId",
"service.name": "opbeans-java",
"tags": Array [
"tags",
],
},
Object {
"@timestamp": "2021-06-16T09:01:00.000Z",
"event.action": "open",
"event.kind": "event",
"kibana.rac.alert.duration.us": 0,
"kibana.rac.alert.id": "opbeans-node",
"kibana.rac.alert.owner": "consumer",
"kibana.rac.alert.producer": "producer",
"kibana.rac.alert.start": "2021-06-16T09:01:00.000Z",
"kibana.rac.alert.status": "open",
"kibana.space_ids": Array [
"spaceId",
],
"rule.category": "ruleTypeName",
"rule.id": "ruleTypeId",
"rule.name": "name",
"rule.uuid": "alertId",
"service.name": "opbeans-node",
"tags": Array [
"tags",
],
},
Object {
"@timestamp": "2021-06-16T09:01:00.000Z",
"event.action": "open",
@ -324,7 +280,7 @@ describe('createLifecycleRuleTypeFactory', () => {
const evaluationDocuments = documents.filter((doc) => doc['event.kind'] === 'event');
const alertDocuments = documents.filter((doc) => doc['event.kind'] === 'signal');
expect(evaluationDocuments.length).toBe(2);
expect(evaluationDocuments.length).toBe(0);
expect(alertDocuments.length).toBe(2);
expect(

15
x-pack/plugins/rule_registry/server/utils/get_rule_executor_data.ts
View file

@ -14,7 +14,6 @@ import {
RULE_UUID,
TAGS,
} from '../../common/technical_rule_data_field_names';
import { AlertTypeExecutor, AlertTypeWithExecutor } from '../types';
export interface RuleExecutorData {
[RULE_CATEGORY]: string;
@ -25,20 +24,6 @@ export interface RuleExecutorData {
[TAGS]: string[];
}
export function getRuleExecutorData(
type: AlertTypeWithExecutor<any, any, any>,
options: Parameters<AlertTypeExecutor>[0]
) {
return {
[RULE_ID]: type.id,
[RULE_UUID]: options.alertId,
[RULE_CATEGORY]: type.name,
[RULE_NAME]: options.name,
[TAGS]: options.tags,
[PRODUCER]: type.producer,
};
}
export function getRuleData(options: AlertExecutorOptions<any, any, any, any, any>) {
return {
[RULE_ID]: options.rule.ruleTypeId,