maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Move exceptions builder to lists plugin (#94002)

Browse source 
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main 2021-03-09 13:39:23 -08:00 committed by GitHub
parent ff2b3beaa8
commit 3992ed13db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 149 additions and 113 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

156
x-pack/plugins/security_solution/common/detection_engine/build_exceptions_filter.test.ts → x-pack/plugins/lists/common/exceptions/build_exceptions_filter.test.ts
View file

@ -5,41 +5,35 @@
* 2.0.
*/
import { getEntryMatchExcludeMock, getEntryMatchMock } from '../schemas/types/entry_match.mock';
import {
getEntryMatchMock,
getEntryMatchExcludeMock,
} from '../../../lists/common/schemas/types/entry_match.mock';
import {
getEntryMatchAnyMock,
getEntryMatchAnyExcludeMock,
} from '../../../lists/common/schemas/types/entry_match_any.mock';
getEntryMatchAnyMock,
} from '../schemas/types/entry_match_any.mock';
import { getEntryExistsExcludedMock, getEntryExistsMock } from '../schemas/types/entry_exists.mock';
import {
getEntryExistsMock,
getEntryExistsExcludedMock,
} from '../../../lists/common/schemas/types/entry_exists.mock';
import {
getEntryNestedMock,
getEntryNestedExcludeMock,
getEntryNestedMixedEntries,
} from '../../../lists/common/schemas/types/entry_nested.mock';
getEntryNestedMock,
} from '../schemas/types/entry_nested.mock';
import {
getExceptionListItemSchemaMock,
getExceptionListItemSchemaXMock,
} from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
} from '../schemas/response/exception_list_item_schema.mock';
import { EntryMatchAny, ExceptionListItemSchema } from '../schemas';
import {
ExceptionItemSansLargeValueLists,
buildExceptionFilter,
buildExceptionItemFilter,
buildExclusionClause,
buildExistsClause,
buildMatchAnyClause,
buildMatchClause,
buildNestedClause,
createOrClauses,
chunkExceptions,
buildExceptionFilter,
ExceptionItemSansLargeValueLists,
createOrClauses,
} from './build_exceptions_filter';
import { EntryMatchAny, ExceptionListItemSchema } from '../../common/shared_imports';
import { hasLargeValueList } from './utils';
const modifiedGetEntryMatchAnyMock = (): EntryMatchAny => ({
@ -59,22 +53,22 @@ describe('build_exceptions_filter', () => {
describe('buildExceptionFilter', () => {
test('it should return undefined if no exception items', () => {
const booleanFilter = buildExceptionFilter({
lists: [],
excludeExceptions: false,
chunkSize: 1,
excludeExceptions: false,
lists: [],
});
expect(booleanFilter).toBeUndefined();
});
test('it should build a filter given an exception list', () => {
const booleanFilter = buildExceptionFilter({
lists: [getExceptionListItemSchemaMock()],
excludeExceptions: false,
chunkSize: 1,
excludeExceptions: false,
lists: [getExceptionListItemSchemaMock()],
});
expect(booleanFilter).toEqual({
meta: { alias: null, negate: false, disabled: false },
meta: { alias: null, disabled: false, negate: false },
query: {
bool: {
should: [
@ -86,10 +80,10 @@ describe('build_exceptions_filter', () => {
path: 'some.parentField',
query: {
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'some.parentField.nested.field': 'some value' } },
],
minimum_should_match: 1,
},
},
score_mode: 'none',
@ -97,8 +91,8 @@ describe('build_exceptions_filter', () => {
},
{
bool: {
should: [{ match_phrase: { 'some.not.nested.field': 'some value' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'some.not.nested.field': 'some value' } }],
},
},
],
@ -123,15 +117,15 @@ describe('build_exceptions_filter', () => {
entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
};
const exceptionFilter = buildExceptionFilter({
lists: [exceptionItem1, exceptionItem2],
excludeExceptions: true,
chunkSize: 2,
excludeExceptions: true,
lists: [exceptionItem1, exceptionItem2],
});
expect(exceptionFilter).toEqual({
meta: {
alias: null,
negate: true,
disabled: false,
negate: true,
},
query: {
bool: {
@ -201,16 +195,16 @@ describe('build_exceptions_filter', () => {
entries: [{ field: 'file.path', operator: 'included', type: 'match', value: '/safe/path' }],
};
const exceptionFilter = buildExceptionFilter({
lists: [exceptionItem1, exceptionItem2, exceptionItem3],
excludeExceptions: true,
chunkSize: 2,
excludeExceptions: true,
lists: [exceptionItem1, exceptionItem2, exceptionItem3],
});
expect(exceptionFilter).toEqual({
meta: {
alias: null,
negate: true,
disabled: false,
negate: true,
},
query: {
bool: {
@ -298,13 +292,13 @@ describe('build_exceptions_filter', () => {
];
const booleanFilter = buildExceptionFilter({
lists: exceptions,
excludeExceptions: true,
chunkSize: 1,
excludeExceptions: true,
lists: exceptions,
});
expect(booleanFilter).toEqual({
meta: { alias: null, negate: true, disabled: false },
meta: { alias: null, disabled: false, negate: true },
query: {
bool: {
should: [
@ -319,21 +313,23 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: { 'parent.field.host.name': 'some host name' },
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -341,11 +337,11 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -353,19 +349,17 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [{ exists: { field: 'parent.field.host.name' } }],
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
},
},
],
@ -382,21 +376,21 @@ describe('build_exceptions_filter', () => {
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
bool: {
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
},
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
minimum_should_match: 1,
},
},
],
@ -412,16 +406,16 @@ describe('build_exceptions_filter', () => {
bool: {
must_not: {
bool: {
should: [{ exists: { field: 'host.name' } }],
minimum_should_match: 1,
should: [{ exists: { field: 'host.name' } }],
},
},
},
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
],
@ -520,18 +514,18 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'parent.field.host.name': 'some host name' } },
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'parent.field.host.name': 'some host name' } },
],
minimum_should_match: 1,
},
},
],
@ -542,8 +536,8 @@ describe('build_exceptions_filter', () => {
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
],
@ -560,19 +554,21 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'parent.field.host.name': 'some host name' } },
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -580,11 +576,11 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -592,19 +588,17 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [{ exists: { field: 'parent.field.host.name' } }],
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
},
},
],
@ -615,29 +609,29 @@ describe('build_exceptions_filter', () => {
},
{
bool: {
minimum_should_match: 1,
should: [
{
bool: {
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
},
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [{ match_phrase: { 'host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
},
@ -645,7 +639,7 @@ describe('build_exceptions_filter', () => {
{
bool: {
must_not: {
bool: { should: [{ exists: { field: 'host.name' } }], minimum_should_match: 1 },
bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] },
},
},
},
@ -655,7 +649,7 @@ describe('build_exceptions_filter', () => {
{
bool: {
must_not: {
bool: { should: [{ exists: { field: 'host.name' } }], minimum_should_match: 1 },
bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] },
},
},
},
@ -686,6 +680,7 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -693,16 +688,17 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -710,11 +706,11 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -722,19 +718,17 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [{ exists: { field: 'parent.field.host.name' } }],
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
},
},
],
@ -745,29 +739,29 @@ describe('build_exceptions_filter', () => {
},
{
bool: {
minimum_should_match: 1,
should: [
{
bool: {
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some "host" name' } }],
},
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
minimum_should_match: 1,
},
},
{
bool: {
must_not: {
bool: {
should: [{ match_phrase: { 'host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
},
@ -775,7 +769,7 @@ describe('build_exceptions_filter', () => {
{
bool: {
must_not: {
bool: { should: [{ exists: { field: 'host.name' } }], minimum_should_match: 1 },
bool: { minimum_should_match: 1, should: [{ exists: { field: 'host.name' } }] },
},
},
},
@ -905,21 +899,21 @@ describe('build_exceptions_filter', () => {
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
should: [{ match_phrase: { 'host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some host name' } }],
},
},
{
bool: {
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'host.name': 'some other host name' } }],
},
},
],
minimum_should_match: 1,
},
},
},
@ -961,14 +955,14 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
},
},
{
bool: {
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
},
},
],
@ -992,8 +986,8 @@ describe('build_exceptions_filter', () => {
bool: {
must_not: {
bool: {
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
},
},
},
@ -1002,17 +996,19 @@ describe('build_exceptions_filter', () => {
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'parent.field.host.name': 'some host name' } },
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -1020,11 +1016,9 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
@ -1048,25 +1042,27 @@ describe('build_exceptions_filter', () => {
filter: [
{
bool: {
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
minimum_should_match: 1,
should: [{ match_phrase: { 'parent.field.host.name': 'some host name' } }],
},
},
{
bool: {
must_not: {
bool: {
minimum_should_match: 1,
should: [
{
bool: {
minimum_should_match: 1,
should: [
{ match_phrase: { 'parent.field.host.name': 'some host name' } },
],
minimum_should_match: 1,
},
},
{
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -1074,19 +1070,17 @@ describe('build_exceptions_filter', () => {
},
},
],
minimum_should_match: 1,
},
},
],
minimum_should_match: 1,
},
},
},
},
{
bool: {
should: [{ exists: { field: 'parent.field.host.name' } }],
minimum_should_match: 1,
should: [{ exists: { field: 'parent.field.host.name' } }],
},
},
],

25
x-pack/plugins/security_solution/common/detection_engine/build_exceptions_filter.ts → x-pack/plugins/lists/common/exceptions/build_exceptions_filter.ts
View file

@ -9,17 +9,18 @@ import { chunk } from 'lodash/fp';
import { Filter } from '../../../../../src/plugins/data/common';
import {
ExceptionListItemSchema,
CreateExceptionListItemSchema,
EntryExists,
EntryMatch,
EntryMatchAny,
EntryNested,
ExceptionListItemSchema,
entriesExists,
entriesMatch,
entriesMatchAny,
entriesExists,
entriesNested,
EntryExists,
} from '../../../lists/common';
} from '../schemas';
import { BooleanFilter, NestedFilter } from './types';
import { hasLargeValueList } from './utils';
@ -83,8 +84,8 @@ export const buildExceptionFilter = ({
const exceptionFilter: Filter = {
meta: {
alias: null,
negate: excludeExceptions,
disabled: false,
negate: excludeExceptions,
},
query: {
bool: {
@ -108,8 +109,8 @@ export const buildExceptionFilter = ({
return {
meta: {
alias: null,
negate: false,
disabled: false,
negate: false,
},
query: {
bool: {
@ -124,8 +125,8 @@ export const buildExceptionFilter = ({
return {
meta: {
alias: null,
negate: excludeExceptions,
disabled: false,
negate: excludeExceptions,
},
query: {
bool: {
@ -148,6 +149,7 @@ export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
const { field, operator, value } = entry;
const matchClause = {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -155,7 +157,6 @@ export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
},
},
],
minimum_should_match: 1,
},
};
@ -172,6 +173,7 @@ export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
if (value.length === 1) {
return {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -179,16 +181,17 @@ export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
},
},
],
minimum_should_match: 1,
},
};
}
return {
bool: {
minimum_should_match: 1,
should: value.map((val) => {
return {
bool: {
minimum_should_match: 1,
should: [
{
match_phrase: {
@ -196,11 +199,9 @@ export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
},
},
],
minimum_should_match: 1,
},
};
}),
minimum_should_match: 1,
},
};
};
@ -220,6 +221,7 @@ export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
const { field, operator } = entry;
const existsClause = {
bool: {
minimum_should_match: 1,
should: [
{
exists: {
@ -227,7 +229,6 @@ export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
},
},
],
minimum_should_match: 1,
},
};

8
x-pack/plugins/lists/common/exceptions/index.ts Normal file
View file

@ -0,0 +1,8 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export * from './build_exceptions_filter';

24
x-pack/plugins/lists/common/exceptions/types.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
export interface BooleanFilter {
bool: {
must?: unknown | unknown[];
must_not?: unknown | unknown[];
should?: unknown[];
filter?: unknown | unknown[];
minimum_should_match?: number;
};
}
export interface NestedFilter {
nested: {
path: string;
query: unknown | unknown[];
score_mode: string;
};
}

24
x-pack/plugins/lists/common/exceptions/utils.ts Normal file
View file

@ -0,0 +1,24 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { CreateExceptionListItemSchema, EntriesArray, ExceptionListItemSchema } from '../schemas';
export const hasLargeValueItem = (
exceptionItems: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>
): boolean => {
return exceptionItems.some((exceptionItem) => hasLargeValueList(exceptionItem.entries));
};
export const hasLargeValueList = (entries: EntriesArray): boolean => {
const found = entries.filter(({ type }) => type === 'list');
return found.length > 0;
};
export const hasNestedEntry = (entries: EntriesArray): boolean => {
const found = entries.filter(({ type }) => type === 'nested');
return found.length > 0;
};

2
x-pack/plugins/lists/common/shared_exports.ts
View file

@ -48,4 +48,6 @@ export {
OsTypeArray,
} from './schemas';
export { buildExceptionFilter } from './exceptions';
export { ENDPOINT_LIST_ID, ENDPOINT_TRUSTED_APPS_LIST_ID } from './constants';

2
x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
View file

@ -16,7 +16,7 @@ import {
CreateExceptionListItemSchema,
} from '../../../lists/common/schemas';
import { ESBoolQuery } from '../typed_json';
import { buildExceptionFilter } from './build_exceptions_filter';
import { buildExceptionFilter } from '../shared_imports';
import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
export const getQueryFilter = (

18
x-pack/plugins/security_solution/common/detection_engine/types.ts
View file

@ -56,21 +56,3 @@ export interface EqlSearchResponse<T> {
events?: Array<BaseHit<T>>;
};
}
export interface BooleanFilter {
bool: {
must?: unknown | unknown[];
must_not?: unknown | unknown[];
should?: unknown[];
filter?: unknown | unknown[];
minimum_should_match?: number;
};
}
export interface NestedFilter {
nested: {
path: string;
query: unknown | unknown[];
score_mode: string;
};
}

1
x-pack/plugins/security_solution/common/shared_imports.ts
View file

@ -47,4 +47,5 @@ export {
ENDPOINT_TRUSTED_APPS_LIST_ID,
osTypeArray,
OsTypeArray,
buildExceptionFilter,
} from '../../lists/common';

2
x-pack/plugins/security_solution/server/lib/machine_learning/index.ts
View file

@ -7,7 +7,7 @@
import { RequestParams } from '@elastic/elasticsearch';
import { buildExceptionFilter } from '../../../common/detection_engine/build_exceptions_filter';
import { buildExceptionFilter } from '../../../common/shared_imports';
import { ExceptionListItemSchema } from '../../../../lists/common';
import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server';
import { SearchResponse } from '../types';