[SIEM][Detection Engine] - Update list values in REST interfaces (#62320)

Summary
- #60022
- Follow up on #60171
- Modifies boolean filters to enum of "included" and "excluded"
- Adds operator types of enum "match", "match_all", "list", and "exists"
- Adds values properties to include those for "list"
- DOES NOT FILTER ON THE VALUES JUST YET (That will be a follow on PR)
This commit is contained in:
Yara Tercero 2020-04-08 10:46:06 -04:00 committed by GitHub
parent 1af82c709f
commit 5d34697ea5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 583 additions and 301 deletions

View file

@ -450,25 +450,31 @@ export const getResult = (): RuleAlertType => ({
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -141,25 +141,31 @@ export const getOutputRuleAlertForRest = (): Omit<
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -74,25 +74,31 @@ export const ruleOutput: RulesSchema = {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -1561,25 +1561,31 @@ describe('add prepackaged rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -1526,25 +1526,31 @@ describe('create rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -1747,25 +1747,31 @@ describe('import rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -1229,25 +1229,31 @@ describe('patch rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -1263,25 +1269,28 @@ describe('patch rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
],
},
],
},

View file

@ -66,25 +66,31 @@ export const getBaseResponsePayload = (anchorDate: string = ANCHOR_DATE): RulesS
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -154,13 +154,34 @@ export const note = t.string;
// NOTE: Experimental list support not being shipped currently and behind a feature flag
// TODO: Remove this comment once we lists have passed testing and is ready for the release
export const boolean_operator = t.keyof({ and: null, 'and not': null });
export const list_type = t.keyof({ value: null }); // TODO: (LIST-FEATURE) Eventually this can include "list" when we support lists CRUD
export const list_value = t.exact(t.type({ name: t.string, type: list_type }));
export const list = t.exact(
t.type({
field: t.string,
boolean_operator,
values: t.array(list_value),
})
export const list_field = t.string;
export const list_values_operator = t.keyof({ included: null, excluded: null });
export const list_values_type = t.keyof({ match: null, match_all: null, list: null, exists: null });
export const list_values = t.exact(
t.intersection([
t.type({
name: t.string,
}),
t.partial({
id: t.string,
description: t.string,
created_at,
}),
])
);
export const list = t.exact(
t.intersection([
t.type({
field: t.string,
values_operator: list_values_operator,
values_type: list_values_type,
}),
t.partial({ values: t.array(list_values) }),
])
);
export const list_and = t.intersection([
list,
t.partial({
and: t.array(list),
}),
]);

View file

@ -126,12 +126,28 @@ export const note = Joi.string();
// NOTE: Experimental list support not being shipped currently and behind a feature flag
// TODO: (LIST-FEATURE) Remove this comment once we lists have passed testing and is ready for the release
export const boolean_operator = Joi.string().valid('and', 'and not');
export const list_type = Joi.string().valid('value'); // TODO: (LIST-FEATURE) Eventually this can be "list" when we support list types
export const list_value = Joi.object({ name: Joi.string().required(), type: list_type.required() });
export const list = Joi.object({
field: Joi.string().required(),
boolean_operator: boolean_operator.required(),
values: Joi.array().items(list_value),
export const list_field = Joi.string();
export const list_values_operator = Joi.string().valid(['included', 'excluded']);
export const list_values_types = Joi.string().valid(['match', 'match_all', 'list', 'exists']);
export const list_values = Joi.object({
name: Joi.string().required(),
id: Joi.string(),
description: Joi.string(),
created_at,
});
export const lists = Joi.array().items(list);
export const list = Joi.object({
field: list_field.required(),
values_operator: list_values_operator.required(),
values_type: list_values_types.required(),
values: Joi.when('values_type', {
is: 'exists',
then: Joi.forbidden(),
otherwise: Joi.array()
.items(list_values)
.required(),
}),
});
export const list_and = Joi.object({
and: Joi.array().items(list),
});
export const lists = Joi.array().items(list.concat(list_and));

View file

@ -23,27 +23,129 @@ describe('lists_default_array', () => {
const payload = [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
{
name: 'mothra',
type: 'value',
},
],
and: [
{
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
test('it should not validate an array of lists that includes a values_operator other than included or excluded', () => {
const payload = [
{
field: 'source.ip',
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'exists',
},
{
field: 'host.hostname',
values_operator: 'jibber jabber',
values_type: 'exists',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "jibber jabber" supplied to "values_operator"',
]);
expect(message.schema).toEqual({});
});
// TODO - this scenario should never come up, as the values key is forbidden when values_type is "exists" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that includes "values" when "values_type" is "exists"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'exists',
values: [
{
name: '127.0.0.1',
},
],
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "match" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that does not include "values" when "values_type" is "match"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "match_all" in the incoming schema - need to find a good way to do this in io-ts
test('it will validate an array of lists that does not include "values" when "values_type" is "match_all"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'match_all',
},
];
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual([]);
expect(message.schema).toEqual(payload);
});
// TODO - this scenario should never come up, as the values key is required when values_type is "list" in the incoming schema - need to find a good way to do this in io-ts
test('it should not validate an array of lists that does not include "values" when "values_type" is "list"', () => {
const payload = [
{
field: 'host.name',
values_operator: 'excluded',
values_type: 'list',
},
];
const decoded = ListsDefaultArray.decode(payload);
@ -57,11 +159,11 @@ describe('lists_default_array', () => {
const payload = [
{
field: 'source.ip',
boolean_operator: 'and',
values_operator: 'included',
values_type: 'exists',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
},
@ -70,7 +172,10 @@ describe('lists_default_array', () => {
const decoded = ListsDefaultArray.decode(payload);
const message = pipe(decoded, foldLeftRight);
expect(getPaths(left(message.errors))).toEqual(['Invalid value "5" supplied to ""']);
expect(getPaths(left(message.errors))).toEqual([
'Invalid value "5" supplied to ""',
'Invalid value "5" supplied to ""',
]);
expect(message.schema).toEqual({});
});

View file

@ -7,10 +7,10 @@
import * as t from 'io-ts';
import { Either } from 'fp-ts/lib/Either';
import { list } from '../response/schemas';
import { list_and as listAnd } from '../response/schemas';
export type ListsDefaultArrayC = t.Type<List[], List[], unknown>;
type List = t.TypeOf<typeof list>;
type List = t.TypeOf<typeof listAnd>;
/**
* Types the ListsDefaultArray as:
@ -18,9 +18,9 @@ type List = t.TypeOf<typeof list>;
*/
export const ListsDefaultArray: ListsDefaultArrayC = new t.Type<List[], List[], unknown>(
'listsWithDefaultArray',
t.array(list).is,
t.array(listAnd).is,
(input): Either<t.Errors, List[]> =>
input == null ? t.success([]) : t.array(list).decode(input),
input == null ? t.success([]) : t.array(listAnd).decode(input),
t.identity
);

View file

@ -1552,25 +1552,28 @@ describe('create rules schema', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
],
},
],
},

View file

@ -82,25 +82,31 @@ describe('getExportAll', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -90,25 +90,31 @@ describe('get_export_by_object_ids', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -212,25 +218,31 @@ describe('get_export_by_object_ids', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -3,21 +3,28 @@
"lists": [
{
"field": "source.ip",
"boolean_operator": "and",
"values": [
{
"name": "127.0.0.1",
"type": "value"
}
]
"values_operator": "excluded",
"values_type": "exists"
},
{
"field": "host.name",
"boolean_operator": "and not",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "rock01",
"type": "value"
"name": "rock01"
}
],
"and": [
{
"field": "host.id",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "123456"
}
]
}
]
}

View file

@ -9,25 +9,31 @@
"lists": [
{
"field": "source.ip",
"boolean_operator": "and",
"values": [
{
"name": "127.0.0.1",
"type": "value"
}
]
"values_operator": "included",
"values_type": "exists"
},
{
"field": "host.name",
"boolean_operator": "and not",
"values_operator": "excluded",
"values_type": "match",
"values": [
{
"name": "rock01",
"type": "value"
},
"name": "rock01"
}
],
"and": [
{
"name": "mothra",
"type": "value"
"field": "host.id",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "123"
},
{
"name": "678"
}
]
}
]
}

View file

@ -9,21 +9,28 @@
"lists": [
{
"field": "source.ip",
"boolean_operator": "and",
"values": [
{
"name": "127.0.0.1",
"type": "value"
}
]
"values_operator": "excluded",
"values_type": "exists"
},
{
"field": "host.name",
"boolean_operator": "and not",
"values_operator": "included",
"values_type": "match",
"values": [
{
"name": "rock01",
"type": "value"
"name": "rock01"
}
],
"and": [
{
"field": "host.id",
"values_operator": "included",
"values_type": "match_all",
"values": [
{
"name": "123456"
}
]
}
]
}

View file

@ -47,25 +47,31 @@ export const sampleRuleAlertParams = (
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -93,25 +93,31 @@ describe('buildBulkBody', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -213,25 +219,31 @@ describe('buildBulkBody', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -331,25 +343,31 @@ describe('buildBulkBody', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -442,25 +460,31 @@ describe('buildBulkBody', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},

View file

@ -82,25 +82,31 @@ describe('buildRule', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -159,25 +165,31 @@ describe('buildRule', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},
@ -235,25 +247,31 @@ describe('buildRule', () => {
lists: [
{
field: 'source.ip',
boolean_operator: 'and',
values: [
{
name: '127.0.0.1',
type: 'value',
},
],
values_operator: 'included',
values_type: 'exists',
},
{
field: 'host.name',
boolean_operator: 'and not',
values_operator: 'excluded',
values_type: 'match',
values: [
{
name: 'rock01',
type: 'value',
},
],
and: [
{
name: 'mothra',
type: 'value',
field: 'host.id',
values_operator: 'included',
values_type: 'match_all',
values: [
{
name: '123',
},
{
name: '678',
},
],
},
],
},