[Alerting] add functional tests for index threshold alertType (#60597)

resolves https://github.com/elastic/kibana/issues/58902
This commit is contained in:
Patrick Mueller 2020-03-19 18:29:26 -04:00 committed by GitHub
parent d1aaa4430a
commit d5989e8baa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 447 additions and 49 deletions

View file

@ -113,6 +113,7 @@ export function getAlertType(service: Service): AlertType {
timeWindowUnit: params.timeWindowUnit,
interval: undefined,
};
// console.log(`index_threshold: query: ${JSON.stringify(queryParams, null, 4)}`);
const result = await service.indexThreshold.timeSeriesQuery({
logger,
callCluster,
@ -121,6 +122,7 @@ export function getAlertType(service: Service): AlertType {
logger.debug(`alert ${ID}:${alertId} "${name}" query result: ${JSON.stringify(result)}`);
const groupResults = result.results || [];
// console.log(`index_threshold: response: ${JSON.stringify(groupResults, null, 4)}`);
for (const groupResult of groupResults) {
const instanceId = groupResult.group;
const value = groupResult.metrics[0][1];

View file

@ -4,20 +4,18 @@
* you may not use this file except in compliance with the Elastic License.
*/
export const ES_TEST_INDEX_NAME = '.kibaka-alerting-test-data';
export const ES_TEST_INDEX_NAME = '.kibana-alerting-test-data';
export class ESTestIndexTool {
private readonly es: any;
private readonly retry: any;
constructor(es: any, retry: any) {
this.es = es;
this.retry = retry;
}
constructor(
private readonly es: any,
private readonly retry: any,
private readonly index: string = ES_TEST_INDEX_NAME
) {}
async setup() {
return await this.es.indices.create({
index: ES_TEST_INDEX_NAME,
index: this.index,
body: {
mappings: {
properties: {
@ -56,12 +54,13 @@ export class ESTestIndexTool {
}
async destroy() {
return await this.es.indices.delete({ index: ES_TEST_INDEX_NAME, ignore: [404] });
return await this.es.indices.delete({ index: this.index, ignore: [404] });
}
async search(source: string, reference: string) {
return await this.es.search({
index: ES_TEST_INDEX_NAME,
index: this.index,
size: 1000,
body: {
query: {
bool: {
@ -86,7 +85,7 @@ export class ESTestIndexTool {
async waitForDocs(source: string, reference: string, numDocs: number = 1) {
return await this.retry.try(async () => {
const searchResult = await this.search(source, reference);
if (searchResult.hits.total.value !== numDocs) {
if (searchResult.hits.total.value < numDocs) {
throw new Error(`Expected ${numDocs} but received ${searchResult.hits.total.value}.`);
}
return searchResult.hits.hits;

View file

@ -0,0 +1,398 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import expect from '@kbn/expect';
import { Spaces } from '../../../../scenarios';
import { FtrProviderContext } from '../../../../../common/ftr_provider_context';
import {
ESTestIndexTool,
ES_TEST_INDEX_NAME,
getUrlPrefix,
ObjectRemover,
} from '../../../../../common/lib';
import { createEsDocuments } from './create_test_data';
const ALERT_TYPE_ID = '.index-threshold';
const ACTION_TYPE_ID = '.index';
const ES_TEST_INDEX_SOURCE = 'builtin-alert:index-threshold';
const ES_TEST_INDEX_REFERENCE = '-na-';
const ES_TEST_OUTPUT_INDEX_NAME = `${ES_TEST_INDEX_NAME}-output`;
const ALERT_INTERVALS_TO_WRITE = 5;
const ALERT_INTERVAL_SECONDS = 3;
const ALERT_INTERVAL_MILLIS = ALERT_INTERVAL_SECONDS * 1000;
// eslint-disable-next-line import/no-default-export
export default function alertTests({ getService }: FtrProviderContext) {
const supertest = getService('supertest');
const retry = getService('retry');
const es = getService('legacyEs');
const esTestIndexTool = new ESTestIndexTool(es, retry);
const esTestIndexToolOutput = new ESTestIndexTool(es, retry, ES_TEST_OUTPUT_INDEX_NAME);
describe('alert', async () => {
let endDate: string;
let actionId: string;
const objectRemover = new ObjectRemover(supertest);
beforeEach(async () => {
await esTestIndexTool.destroy();
await esTestIndexTool.setup();
await esTestIndexToolOutput.destroy();
await esTestIndexToolOutput.setup();
actionId = await createAction(supertest, objectRemover);
// write documents in the future, figure out the end date
const endDateMillis = Date.now() + (ALERT_INTERVALS_TO_WRITE - 1) * ALERT_INTERVAL_MILLIS;
endDate = new Date(endDateMillis).toISOString();
// write documents from now to the future end date in 3 groups
createEsDocumentsInGroups(3);
});
afterEach(async () => {
await objectRemover.removeAll();
await esTestIndexTool.destroy();
await esTestIndexToolOutput.destroy();
});
// The tests below create two alerts, one that will fire, one that will
// never fire; the tests ensure the ones that should fire, do fire, and
// those that shouldn't fire, do not fire.
it('runs correctly: count all < >', async () => {
await createAlert({
name: 'never fire',
aggType: 'count',
groupBy: 'all',
thresholdComparator: '<',
threshold: [0],
});
await createAlert({
name: 'always fire',
aggType: 'count',
groupBy: 'all',
thresholdComparator: '>',
threshold: [-1],
});
const docs = await waitForDocs(2);
for (const doc of docs) {
const { group } = doc._source;
const { name, value, title, message } = doc._source.params;
expect(name).to.be('always fire');
expect(group).to.be('all documents');
// we'll check title and message in this test, but not subsequent ones
expect(title).to.be('alert always fire group all documents exceeded threshold');
const expectedPrefix = `alert always fire group all documents value ${value} exceeded threshold count > -1 over`;
const messagePrefix = message.substr(0, expectedPrefix.length);
expect(messagePrefix).to.be(expectedPrefix);
}
});
it('runs correctly: count grouped <= =>', async () => {
// create some more documents in the first group
createEsDocumentsInGroups(1);
await createAlert({
name: 'never fire',
aggType: 'count',
groupBy: 'top',
termField: 'group',
termSize: 2,
thresholdComparator: '<=',
threshold: [-1],
});
await createAlert({
name: 'always fire',
aggType: 'count',
groupBy: 'top',
termField: 'group',
termSize: 2, // two actions will fire each interval
thresholdComparator: '>=',
threshold: [0],
});
const docs = await waitForDocs(4);
let inGroup0 = 0;
for (const doc of docs) {
const { group } = doc._source;
const { name } = doc._source.params;
expect(name).to.be('always fire');
if (group === 'group-0') inGroup0++;
}
// there should be 2 docs in group-0, rando split between others
expect(inGroup0).to.be(2);
});
it('runs correctly: sum all between', async () => {
// create some more documents in the first group
createEsDocumentsInGroups(1);
await createAlert({
name: 'never fire',
aggType: 'sum',
aggField: 'testedValue',
groupBy: 'all',
thresholdComparator: 'between',
threshold: [-2, -1],
});
await createAlert({
name: 'always fire',
aggType: 'sum',
aggField: 'testedValue',
groupBy: 'all',
thresholdComparator: 'between',
threshold: [0, 1000000],
});
const docs = await waitForDocs(2);
for (const doc of docs) {
const { name } = doc._source.params;
expect(name).to.be('always fire');
}
});
it('runs correctly: avg all', async () => {
// create some more documents in the first group
createEsDocumentsInGroups(1);
await createAlert({
name: 'never fire',
aggType: 'avg',
aggField: 'testedValue',
groupBy: 'all',
thresholdComparator: '<',
threshold: [0],
});
await createAlert({
name: 'always fire',
aggType: 'avg',
aggField: 'testedValue',
groupBy: 'all',
thresholdComparator: '>=',
threshold: [0],
});
const docs = await waitForDocs(4);
for (const doc of docs) {
const { name } = doc._source.params;
expect(name).to.be('always fire');
}
});
it('runs correctly: max grouped', async () => {
// create some more documents in the first group
createEsDocumentsInGroups(1);
await createAlert({
name: 'never fire',
aggType: 'max',
aggField: 'testedValue',
groupBy: 'top',
termField: 'group',
termSize: 2,
thresholdComparator: '<',
threshold: [0],
});
await createAlert({
name: 'always fire',
aggType: 'max',
aggField: 'testedValue',
groupBy: 'top',
termField: 'group',
termSize: 2, // two actions will fire each interval
thresholdComparator: '>=',
threshold: [0],
});
const docs = await waitForDocs(4);
let inGroup2 = 0;
for (const doc of docs) {
const { group } = doc._source;
const { name } = doc._source.params;
expect(name).to.be('always fire');
if (group === 'group-2') inGroup2++;
}
// there should be 2 docs in group-2, rando split between others
expect(inGroup2).to.be(2);
});
it('runs correctly: min grouped', async () => {
// create some more documents in the first group
createEsDocumentsInGroups(1);
await createAlert({
name: 'never fire',
aggType: 'min',
aggField: 'testedValue',
groupBy: 'top',
termField: 'group',
termSize: 2,
thresholdComparator: '<',
threshold: [0],
});
await createAlert({
name: 'always fire',
aggType: 'min',
aggField: 'testedValue',
groupBy: 'top',
termField: 'group',
termSize: 2, // two actions will fire each interval
thresholdComparator: '>=',
threshold: [0],
});
const docs = await waitForDocs(4);
let inGroup0 = 0;
for (const doc of docs) {
const { group } = doc._source;
const { name } = doc._source.params;
expect(name).to.be('always fire');
if (group === 'group-0') inGroup0++;
}
// there should be 2 docs in group-0, rando split between others
expect(inGroup0).to.be(2);
});
async function createEsDocumentsInGroups(groups: number) {
await createEsDocuments(
es,
esTestIndexTool,
endDate,
ALERT_INTERVALS_TO_WRITE,
ALERT_INTERVAL_MILLIS,
groups
);
}
async function waitForDocs(count: number): Promise<any[]> {
return await esTestIndexToolOutput.waitForDocs(
ES_TEST_INDEX_SOURCE,
ES_TEST_INDEX_REFERENCE,
count
);
}
interface CreateAlertParams {
name: string;
aggType: string;
aggField?: string;
groupBy: 'all' | 'top';
termField?: string;
termSize?: number;
thresholdComparator: string;
threshold: number[];
}
async function createAlert(params: CreateAlertParams): Promise<string> {
const action = {
id: actionId,
group: 'threshold met',
params: {
documents: [
{
source: ES_TEST_INDEX_SOURCE,
reference: ES_TEST_INDEX_REFERENCE,
params: {
name: '{{{alertName}}}',
value: '{{{context.value}}}',
title: '{{{context.title}}}',
message: '{{{context.message}}}',
},
date: '{{{context.date}}}',
// TODO: I wanted to write the alert value here, but how?
// We only mustache interpolate string values ...
// testedValue: '{{{context.value}}}',
group: '{{{context.group}}}',
},
],
},
};
const { statusCode, body: createdAlert } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/alert`)
.set('kbn-xsrf', 'foo')
.send({
name: params.name,
consumer: 'function test',
enabled: true,
alertTypeId: ALERT_TYPE_ID,
schedule: { interval: `${ALERT_INTERVAL_SECONDS}s` },
actions: [action],
params: {
index: ES_TEST_INDEX_NAME,
timeField: 'date',
aggType: params.aggType,
aggField: params.aggField,
groupBy: params.groupBy,
termField: params.termField,
termSize: params.termSize,
timeWindowSize: ALERT_INTERVAL_SECONDS * 5,
timeWindowUnit: 's',
thresholdComparator: params.thresholdComparator,
threshold: params.threshold,
},
});
// will print the error body, if an error occurred
// if (statusCode !== 200) console.log(createdAlert);
expect(statusCode).to.be(200);
const alertId = createdAlert.id;
objectRemover.add(Spaces.space1.id, alertId, 'alert');
return alertId;
}
});
}
async function createAction(supertest: any, objectRemover: ObjectRemover): Promise<string> {
const { statusCode, body: createdAction } = await supertest
.post(`${getUrlPrefix(Spaces.space1.id)}/api/action`)
.set('kbn-xsrf', 'foo')
.send({
name: 'index action for index threshold FT',
actionTypeId: ACTION_TYPE_ID,
config: {
index: ES_TEST_OUTPUT_INDEX_NAME,
},
secrets: {},
});
// will print the error body, if an error occurred
// if (statusCode !== 200) console.log(createdAction);
expect(statusCode).to.be(200);
const actionId = createdAction.id;
objectRemover.add(Spaces.space1.id, actionId, 'action');
return actionId;
}

View file

@ -8,53 +8,50 @@ import { times } from 'lodash';
import { v4 as uuid } from 'uuid';
import { ESTestIndexTool, ES_TEST_INDEX_NAME } from '../../../../../common/lib';
// date to start writing data
export const START_DATE = '2020-01-01T00:00:00Z';
// default end date
export const END_DATE = '2020-01-01T00:00:00Z';
const DOCUMENT_SOURCE = 'queryDataEndpointTests';
export const DOCUMENT_SOURCE = 'queryDataEndpointTests';
export const DOCUMENT_REFERENCE = '-na-';
// Create a set of es documents to run the queries against.
// Will create 2 documents for each interval.
// Will create `groups` documents for each interval.
// The difference between the dates of the docs will be intervalMillis.
// The date of the last documents will be startDate - intervalMillis / 2.
// So there will be 2 documents written in the middle of each interval range.
// The data value written to each doc is a power of 2, with 2^0 as the value
// of the last documents, the values increasing for older documents. The
// second document for each time value will be power of 2 + 1
// So the documents will be written in the middle of each interval range.
// The data value written to each doc is a power of 2 + the group index, with
// 2^0 as the value of the last documents, the values increasing for older
// documents.
export async function createEsDocuments(
es: any,
esTestIndexTool: ESTestIndexTool,
startDate: string = START_DATE,
endDate: string = END_DATE,
intervals: number = 1,
intervalMillis: number = 1000
intervalMillis: number = 1000,
groups: number = 2
) {
const totalDocuments = intervals * 2;
const startDateMillis = Date.parse(startDate) - intervalMillis / 2;
const endDateMillis = Date.parse(endDate) - intervalMillis / 2;
times(intervals, interval => {
const date = startDateMillis - interval * intervalMillis;
const date = endDateMillis - interval * intervalMillis;
// base value for each window is 2^window
// base value for each window is 2^interval
const testedValue = 2 ** interval;
// don't need await on these, wait at the end of the function
createEsDocument(es, '-na-', date, testedValue, 'groupA');
createEsDocument(es, '-na-', date, testedValue + 1, 'groupB');
times(groups, group => {
createEsDocument(es, date, testedValue + group, `group-${group}`);
});
});
await esTestIndexTool.waitForDocs(DOCUMENT_SOURCE, '-na-', totalDocuments);
const totalDocuments = intervals * groups;
await esTestIndexTool.waitForDocs(DOCUMENT_SOURCE, DOCUMENT_REFERENCE, totalDocuments);
}
async function createEsDocument(
es: any,
reference: string,
epochMillis: number,
testedValue: number,
group: string
) {
async function createEsDocument(es: any, epochMillis: number, testedValue: number, group: string) {
const document = {
source: DOCUMENT_SOURCE,
reference,
reference: DOCUMENT_REFERENCE,
date: new Date(epochMillis).toISOString(),
testedValue,
group,
@ -65,6 +62,7 @@ async function createEsDocument(
index: ES_TEST_INDEX_NAME,
body: document,
});
// console.log(`writing document to ${ES_TEST_INDEX_NAME}:`, JSON.stringify(document, null, 4));
if (response.result !== 'created') {
throw new Error(`document not created: ${JSON.stringify(response)}`);

View file

@ -12,5 +12,6 @@ export default function alertingTests({ loadTestFile }: FtrProviderContext) {
loadTestFile(require.resolve('./time_series_query_endpoint'));
loadTestFile(require.resolve('./fields_endpoint'));
loadTestFile(require.resolve('./indices_endpoint'));
loadTestFile(require.resolve('./alert'));
});
}

View file

@ -39,12 +39,12 @@ const START_DATE_MINUS_2INTERVALS = getStartDate(-2 * INTERVAL_MILLIS);
are offset from the top of the minute by 30 seconds, the queries always
run from the top of the hour.
{ "date":"2019-12-31T23:59:30.000Z", "testedValue":1, "group":"groupA" }
{ "date":"2019-12-31T23:59:30.000Z", "testedValue":2, "group":"groupB" }
{ "date":"2019-12-31T23:58:30.000Z", "testedValue":2, "group":"groupA" }
{ "date":"2019-12-31T23:58:30.000Z", "testedValue":3, "group":"groupB" }
{ "date":"2019-12-31T23:57:30.000Z", "testedValue":4, "group":"groupA" }
{ "date":"2019-12-31T23:57:30.000Z", "testedValue":5, "group":"groupB" }
{ "date":"2019-12-31T23:59:30.000Z", "testedValue":1, "group":"group-0" }
{ "date":"2019-12-31T23:59:30.000Z", "testedValue":2, "group":"group-1" }
{ "date":"2019-12-31T23:58:30.000Z", "testedValue":2, "group":"group-0" }
{ "date":"2019-12-31T23:58:30.000Z", "testedValue":3, "group":"group-1" }
{ "date":"2019-12-31T23:57:30.000Z", "testedValue":4, "group":"group-0" }
{ "date":"2019-12-31T23:57:30.000Z", "testedValue":5, "group":"group-1" }
*/
// eslint-disable-next-line import/no-default-export
@ -162,7 +162,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
const expected = {
results: [
{
group: 'groupA',
group: 'group-0',
metrics: [
[START_DATE_MINUS_2INTERVALS, 1],
[START_DATE_MINUS_1INTERVALS, 2],
@ -170,7 +170,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
],
},
{
group: 'groupB',
group: 'group-1',
metrics: [
[START_DATE_MINUS_2INTERVALS, 1],
[START_DATE_MINUS_1INTERVALS, 2],
@ -197,7 +197,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
const expected = {
results: [
{
group: 'groupB',
group: 'group-1',
metrics: [
[START_DATE_MINUS_2INTERVALS, 5 / 1],
[START_DATE_MINUS_1INTERVALS, (5 + 3) / 2],
@ -205,7 +205,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
],
},
{
group: 'groupA',
group: 'group-0',
metrics: [
[START_DATE_MINUS_2INTERVALS, 4 / 1],
[START_DATE_MINUS_1INTERVALS, (4 + 2) / 2],
@ -230,7 +230,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
});
const result = await runQueryExpect(query, 200);
expect(result.results.length).to.be(1);
expect(result.results[0].group).to.be('groupB');
expect(result.results[0].group).to.be('group-1');
});
it('should return correct sorted group for min', async () => {
@ -245,7 +245,7 @@ export default function timeSeriesQueryEndpointTests({ getService }: FtrProvider
});
const result = await runQueryExpect(query, 200);
expect(result.results.length).to.be(1);
expect(result.results[0].group).to.be('groupA');
expect(result.results[0].group).to.be('group-0');
});
it('should return an error when passed invalid input', async () => {