[Security Solution][Detection Engine] - Improve DE query build times for large lists (#85051)
## Summary This PR addresses the following issues: - https://github.com/elastic/kibana/issues/76979 - https://github.com/elastic/kibana/issues/82267 - removal of unused lucene exceptions logic
This commit is contained in:
Yara Tercero
GitHub
parent
e8a8f20932
commit
21ea4f7a6f
|
|
@ -36,6 +36,7 @@ export const TYPE = 'ip';
|
|||
export const VALUE = '127.0.0.1';
|
||||
export const VALUE_2 = '255.255.255';
|
||||
export const NAMESPACE_TYPE = 'single';
|
||||
export const NESTED_FIELD = 'parent.field';
|
||||
|
||||
// Exception List specific
|
||||
export const ID = 'uuid_here';
|
||||
|
|
|
|||
|
|
@ -43,6 +43,10 @@ export const getExceptionListItemSchemaMock = (): ExceptionListItemSchema => ({
|
|||
updated_by: USER,
|
||||
});
|
||||
|
||||
export const getExceptionListItemSchemaXMock = (count = 1): ExceptionListItemSchema[] => {
|
||||
return new Array(count).fill(null).map(() => getExceptionListItemSchemaMock());
|
||||
};
|
||||
|
||||
/**
|
||||
* This is useful for end to end tests where we remove the auto generated parts for comparisons
|
||||
* such as created_at, updated_at, and id.
|
||||
|
|
|
|||
|
|
@ -13,3 +13,8 @@ export const getEntryExistsMock = (): EntryExists => ({
|
|||
operator: OPERATOR,
|
||||
type: EXISTS,
|
||||
});
|
||||
|
||||
export const getEntryExistsExcludedMock = (): EntryExists => ({
|
||||
...getEntryExistsMock(),
|
||||
operator: 'excluded',
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,3 +14,8 @@ export const getEntryMatchMock = (): EntryMatch => ({
|
|||
type: MATCH,
|
||||
value: ENTRY_VALUE,
|
||||
});
|
||||
|
||||
export const getEntryMatchExcludeMock = (): EntryMatch => ({
|
||||
...getEntryMatchMock(),
|
||||
operator: 'excluded',
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,3 +14,9 @@ export const getEntryMatchAnyMock = (): EntryMatchAny => ({
|
|||
type: MATCH_ANY,
|
||||
value: [ENTRY_VALUE],
|
||||
});
|
||||
|
||||
export const getEntryMatchAnyExcludeMock = (): EntryMatchAny => ({
|
||||
...getEntryMatchAnyMock(),
|
||||
operator: 'excluded',
|
||||
value: [ENTRY_VALUE, 'some other host name'],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,14 +4,25 @@
|
|||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import { FIELD, NESTED } from '../../constants.mock';
|
||||
import { NESTED, NESTED_FIELD } from '../../constants.mock';
|
||||
|
||||
import { EntryNested } from './entry_nested';
|
||||
import { getEntryMatchMock } from './entry_match.mock';
|
||||
import { getEntryMatchAnyMock } from './entry_match_any.mock';
|
||||
import { getEntryMatchExcludeMock, getEntryMatchMock } from './entry_match.mock';
|
||||
import { getEntryMatchAnyExcludeMock, getEntryMatchAnyMock } from './entry_match_any.mock';
|
||||
import { getEntryExistsMock } from './entry_exists.mock';
|
||||
|
||||
export const getEntryNestedMock = (): EntryNested => ({
|
||||
entries: [getEntryMatchMock(), getEntryMatchAnyMock()],
|
||||
field: FIELD,
|
||||
field: NESTED_FIELD,
|
||||
type: NESTED,
|
||||
});
|
||||
|
||||
export const getEntryNestedExcludeMock = (): EntryNested => ({
|
||||
...getEntryNestedMock(),
|
||||
entries: [getEntryMatchExcludeMock(), getEntryMatchAnyExcludeMock()],
|
||||
});
|
||||
|
||||
export const getEntryNestedMixedEntries = (): EntryNested => ({
|
||||
...getEntryNestedMock(),
|
||||
entries: [getEntryMatchMock(), getEntryMatchAnyExcludeMock(), getEntryExistsMock()],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ describe('entriesNested', () => {
|
|||
value: ['some host name'],
|
||||
},
|
||||
],
|
||||
field: 'host.name',
|
||||
field: 'parent.field',
|
||||
type: 'nested',
|
||||
});
|
||||
});
|
||||
|
|
@ -105,7 +105,7 @@ describe('entriesNested', () => {
|
|||
type: 'exists',
|
||||
},
|
||||
],
|
||||
field: 'host.name',
|
||||
field: 'parent.field',
|
||||
type: 'nested',
|
||||
});
|
||||
});
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1,293 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { chunk } from 'lodash/fp';
|
||||
|
||||
import { Filter } from '../../../../../src/plugins/data/common';
|
||||
import {
|
||||
ExceptionListItemSchema,
|
||||
CreateExceptionListItemSchema,
|
||||
EntryMatch,
|
||||
EntryMatchAny,
|
||||
EntryNested,
|
||||
entriesMatch,
|
||||
entriesMatchAny,
|
||||
entriesExists,
|
||||
entriesNested,
|
||||
EntryExists,
|
||||
} from '../../../lists/common';
|
||||
import { BooleanFilter, NestedFilter } from './types';
|
||||
import { hasLargeValueList } from './utils';
|
||||
|
||||
type NonListEntry = EntryMatch | EntryMatchAny | EntryNested | EntryExists;
|
||||
interface ExceptionListItemNonLargeList extends ExceptionListItemSchema {
|
||||
entries: NonListEntry[];
|
||||
}
|
||||
|
||||
interface CreateExceptionListItemNonLargeList extends CreateExceptionListItemSchema {
|
||||
entries: NonListEntry[];
|
||||
}
|
||||
|
||||
export type ExceptionItemSansLargeValueLists =
|
||||
| ExceptionListItemNonLargeList
|
||||
| CreateExceptionListItemNonLargeList;
|
||||
|
||||
export const chunkExceptions = (
|
||||
exceptions: ExceptionItemSansLargeValueLists[],
|
||||
chunkSize: number
|
||||
): ExceptionItemSansLargeValueLists[][] => {
|
||||
return chunk(chunkSize, exceptions);
|
||||
};
|
||||
|
||||
export const buildExceptionItemFilter = (
|
||||
exceptionItem: ExceptionItemSansLargeValueLists
|
||||
): BooleanFilter | NestedFilter => {
|
||||
const { entries } = exceptionItem;
|
||||
|
||||
if (entries.length === 1) {
|
||||
return createInnerAndClauses(entries[0]);
|
||||
} else {
|
||||
return {
|
||||
bool: {
|
||||
filter: entries.map((entry) => createInnerAndClauses(entry)),
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const createOrClauses = (
|
||||
exceptionItems: ExceptionItemSansLargeValueLists[]
|
||||
): Array<BooleanFilter | NestedFilter> => {
|
||||
return exceptionItems.map((exceptionItem) => buildExceptionItemFilter(exceptionItem));
|
||||
};
|
||||
|
||||
export const buildExceptionFilter = ({
|
||||
lists,
|
||||
excludeExceptions,
|
||||
chunkSize,
|
||||
}: {
|
||||
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
|
||||
excludeExceptions: boolean;
|
||||
chunkSize: number;
|
||||
}): Filter | undefined => {
|
||||
// Remove exception items with large value lists. These are evaluated
|
||||
// elsewhere for the moment being.
|
||||
const exceptionsWithoutLargeValueLists = lists.filter(
|
||||
(item): item is ExceptionItemSansLargeValueLists => !hasLargeValueList(item.entries)
|
||||
);
|
||||
|
||||
const exceptionFilter: Filter = {
|
||||
meta: {
|
||||
alias: null,
|
||||
negate: excludeExceptions,
|
||||
disabled: false,
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
should: undefined,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
if (exceptionsWithoutLargeValueLists.length === 0) {
|
||||
return undefined;
|
||||
} else if (exceptionsWithoutLargeValueLists.length <= chunkSize) {
|
||||
const clause = createOrClauses(exceptionsWithoutLargeValueLists);
|
||||
exceptionFilter.query.bool.should = clause;
|
||||
return exceptionFilter;
|
||||
} else {
|
||||
const chunks = chunkExceptions(exceptionsWithoutLargeValueLists, chunkSize);
|
||||
|
||||
const filters = chunks.map<Filter>((exceptionsChunk) => {
|
||||
const orClauses = createOrClauses(exceptionsChunk);
|
||||
|
||||
return {
|
||||
meta: {
|
||||
alias: null,
|
||||
negate: false,
|
||||
disabled: false,
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
should: orClauses,
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const clauses = filters.map<BooleanFilter>(({ query }) => query);
|
||||
|
||||
return {
|
||||
meta: {
|
||||
alias: null,
|
||||
negate: excludeExceptions,
|
||||
disabled: false,
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
should: clauses,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
export const buildExclusionClause = (booleanFilter: BooleanFilter): BooleanFilter => {
|
||||
return {
|
||||
bool: {
|
||||
must_not: booleanFilter,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const buildMatchClause = (entry: EntryMatch): BooleanFilter => {
|
||||
const { field, operator, value } = entry;
|
||||
const matchClause = {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
match_phrase: {
|
||||
[field]: value,
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
};
|
||||
|
||||
if (operator === 'excluded') {
|
||||
return buildExclusionClause(matchClause);
|
||||
} else {
|
||||
return matchClause;
|
||||
}
|
||||
};
|
||||
|
||||
export const getBaseMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
|
||||
const { field, value } = entry;
|
||||
|
||||
if (value.length === 1) {
|
||||
return {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
match_phrase: {
|
||||
[field]: value[0],
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
bool: {
|
||||
should: value.map((val) => {
|
||||
return {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
match_phrase: {
|
||||
[field]: val,
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
};
|
||||
}),
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const buildMatchAnyClause = (entry: EntryMatchAny): BooleanFilter => {
|
||||
const { operator } = entry;
|
||||
const matchAnyClause = getBaseMatchAnyClause(entry);
|
||||
|
||||
if (operator === 'excluded') {
|
||||
return buildExclusionClause(matchAnyClause);
|
||||
} else {
|
||||
return matchAnyClause;
|
||||
}
|
||||
};
|
||||
|
||||
export const buildExistsClause = (entry: EntryExists): BooleanFilter => {
|
||||
const { field, operator } = entry;
|
||||
const existsClause = {
|
||||
bool: {
|
||||
should: [
|
||||
{
|
||||
exists: {
|
||||
field,
|
||||
},
|
||||
},
|
||||
],
|
||||
minimum_should_match: 1,
|
||||
},
|
||||
};
|
||||
|
||||
if (operator === 'excluded') {
|
||||
return buildExclusionClause(existsClause);
|
||||
} else {
|
||||
return existsClause;
|
||||
}
|
||||
};
|
||||
|
||||
const isBooleanFilter = (clause: object): clause is BooleanFilter => {
|
||||
const keys = Object.keys(clause);
|
||||
return keys.includes('bool') != null;
|
||||
};
|
||||
|
||||
export const getBaseNestedClause = (
|
||||
entries: NonListEntry[],
|
||||
parentField: string
|
||||
): BooleanFilter => {
|
||||
if (entries.length === 1) {
|
||||
const [singleNestedEntry] = entries;
|
||||
const innerClause = createInnerAndClauses(singleNestedEntry, parentField);
|
||||
return isBooleanFilter(innerClause) ? innerClause : { bool: {} };
|
||||
}
|
||||
|
||||
return {
|
||||
bool: {
|
||||
filter: entries.map((nestedEntry) => createInnerAndClauses(nestedEntry, parentField)),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const buildNestedClause = (entry: EntryNested): NestedFilter => {
|
||||
const { field, entries } = entry;
|
||||
|
||||
const baseNestedClause = getBaseNestedClause(entries, field);
|
||||
|
||||
return {
|
||||
nested: {
|
||||
path: field,
|
||||
query: baseNestedClause,
|
||||
score_mode: 'none',
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export const createInnerAndClauses = (
|
||||
entry: NonListEntry,
|
||||
parent?: string
|
||||
): BooleanFilter | NestedFilter => {
|
||||
if (entriesExists.is(entry)) {
|
||||
const field = parent != null ? `${parent}.${entry.field}` : entry.field;
|
||||
return buildExistsClause({ ...entry, field });
|
||||
} else if (entriesMatch.is(entry)) {
|
||||
const field = parent != null ? `${parent}.${entry.field}` : entry.field;
|
||||
return buildMatchClause({ ...entry, field });
|
||||
} else if (entriesMatchAny.is(entry)) {
|
||||
const field = parent != null ? `${parent}.${entry.field}` : entry.field;
|
||||
return buildMatchAnyClause({ ...entry, field });
|
||||
} else if (entriesNested.is(entry)) {
|
||||
return buildNestedClause(entry);
|
||||
} else {
|
||||
throw new TypeError(`Unexpected exception entry: ${entry}`);
|
||||
}
|
||||
};
|
||||
|
|
@ -1,788 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
|
||||
import {
|
||||
buildExceptionListQueries,
|
||||
buildExceptionItem,
|
||||
operatorBuilder,
|
||||
buildExists,
|
||||
buildMatch,
|
||||
buildMatchAny,
|
||||
buildEntry,
|
||||
getLanguageBooleanOperator,
|
||||
buildNested,
|
||||
} from './build_exceptions_query';
|
||||
import { EntryNested, EntryMatchAny, EntriesArray } from '../../../lists/common/schemas';
|
||||
import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
|
||||
import { getEntryMatchMock } from '../../../lists/common/schemas/types/entry_match.mock';
|
||||
import { getEntryMatchAnyMock } from '../../../lists/common/schemas/types/entry_match_any.mock';
|
||||
import { getEntryExistsMock } from '../../../lists/common/schemas/types/entry_exists.mock';
|
||||
|
||||
describe('build_exceptions_query', () => {
|
||||
describe('getLanguageBooleanOperator', () => {
|
||||
test('it returns value as uppercase if language is "lucene"', () => {
|
||||
const result = getLanguageBooleanOperator({ language: 'lucene', value: 'not' });
|
||||
|
||||
expect(result).toEqual('NOT');
|
||||
});
|
||||
|
||||
test('it returns value as is if language is "kuery"', () => {
|
||||
const result = getLanguageBooleanOperator({ language: 'kuery', value: 'not' });
|
||||
|
||||
expect(result).toEqual('not');
|
||||
});
|
||||
});
|
||||
|
||||
describe('operatorBuilder', () => {
|
||||
describe('and language is kuery', () => {
|
||||
test('it returns empty string when operator is "included"', () => {
|
||||
const operator = operatorBuilder({ operator: 'included', language: 'kuery' });
|
||||
expect(operator).toEqual('');
|
||||
});
|
||||
test('it returns "not " when operator is "excluded"', () => {
|
||||
const operator = operatorBuilder({ operator: 'excluded', language: 'kuery' });
|
||||
expect(operator).toEqual('not ');
|
||||
});
|
||||
});
|
||||
|
||||
describe('and language is lucene', () => {
|
||||
test('it returns empty string when operator is "included"', () => {
|
||||
const operator = operatorBuilder({ operator: 'included', language: 'lucene' });
|
||||
expect(operator).toEqual('');
|
||||
});
|
||||
test('it returns "NOT " when operator is "excluded"', () => {
|
||||
const operator = operatorBuilder({ operator: 'excluded', language: 'lucene' });
|
||||
expect(operator).toEqual('NOT ');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildExists', () => {
|
||||
describe('kuery', () => {
|
||||
test('it returns formatted wildcard string when operator is "excluded"', () => {
|
||||
const query = buildExists({
|
||||
entry: { ...getEntryExistsMock(), operator: 'excluded' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(query).toEqual('not host.name:*');
|
||||
});
|
||||
test('it returns formatted wildcard string when operator is "included"', () => {
|
||||
const query = buildExists({
|
||||
entry: { ...getEntryExistsMock(), operator: 'included' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(query).toEqual('host.name:*');
|
||||
});
|
||||
});
|
||||
|
||||
describe('lucene', () => {
|
||||
test('it returns formatted wildcard string when operator is "excluded"', () => {
|
||||
const query = buildExists({
|
||||
entry: { ...getEntryExistsMock(), operator: 'excluded' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(query).toEqual('NOT _exists_host.name');
|
||||
});
|
||||
test('it returns formatted wildcard string when operator is "included"', () => {
|
||||
const query = buildExists({
|
||||
entry: { ...getEntryExistsMock(), operator: 'included' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(query).toEqual('_exists_host.name');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMatch', () => {
|
||||
describe('kuery', () => {
|
||||
test('it returns formatted string when operator is "included"', () => {
|
||||
const query = buildMatch({
|
||||
entry: { ...getEntryMatchMock(), operator: 'included' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(query).toEqual('host.name:"some host name"');
|
||||
});
|
||||
test('it returns formatted string when operator is "excluded"', () => {
|
||||
const query = buildMatch({
|
||||
entry: { ...getEntryMatchMock(), operator: 'excluded' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(query).toEqual('not host.name:"some host name"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('lucene', () => {
|
||||
test('it returns formatted string when operator is "included"', () => {
|
||||
const query = buildMatch({
|
||||
entry: { ...getEntryMatchMock(), operator: 'included' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(query).toEqual('host.name:"some host name"');
|
||||
});
|
||||
test('it returns formatted string when operator is "excluded"', () => {
|
||||
const query = buildMatch({
|
||||
entry: { ...getEntryMatchMock(), operator: 'excluded' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(query).toEqual('NOT host.name:"some host name"');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMatchAny', () => {
|
||||
const entryWithIncludedAndNoValues: EntryMatchAny = {
|
||||
...getEntryMatchAnyMock(),
|
||||
field: 'host.name',
|
||||
value: [],
|
||||
};
|
||||
const entryWithIncludedAndOneValue: EntryMatchAny = {
|
||||
...getEntryMatchAnyMock(),
|
||||
field: 'host.name',
|
||||
value: ['some host name'],
|
||||
};
|
||||
const entryWithExcludedAndTwoValues: EntryMatchAny = {
|
||||
...getEntryMatchAnyMock(),
|
||||
field: 'host.name',
|
||||
value: ['some host name', 'auditd'],
|
||||
operator: 'excluded',
|
||||
};
|
||||
|
||||
describe('kuery', () => {
|
||||
test('it returns empty string if given an empty array for "values"', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: entryWithIncludedAndNoValues,
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(exceptionSegment).toEqual('');
|
||||
});
|
||||
|
||||
test('it returns formatted string when "values" includes only one item', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: entryWithIncludedAndOneValue,
|
||||
language: 'kuery',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('host.name:("some host name")');
|
||||
});
|
||||
|
||||
test('it returns formatted string when operator is "included"', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
|
||||
language: 'kuery',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('host.name:("some host name" or "auditd")');
|
||||
});
|
||||
|
||||
test('it returns formatted string when operator is "excluded"', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: entryWithExcludedAndTwoValues,
|
||||
language: 'kuery',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('not host.name:("some host name" or "auditd")');
|
||||
});
|
||||
});
|
||||
|
||||
describe('lucene', () => {
|
||||
test('it returns formatted string when operator is "included"', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
|
||||
language: 'lucene',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('host.name:("some host name" OR "auditd")');
|
||||
});
|
||||
test('it returns formatted string when operator is "excluded"', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: entryWithExcludedAndTwoValues,
|
||||
language: 'lucene',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('NOT host.name:("some host name" OR "auditd")');
|
||||
});
|
||||
test('it returns formatted string when "values" includes only one item', () => {
|
||||
const exceptionSegment = buildMatchAny({
|
||||
entry: entryWithIncludedAndOneValue,
|
||||
language: 'lucene',
|
||||
});
|
||||
|
||||
expect(exceptionSegment).toEqual('host.name:("some host name")');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildNested', () => {
|
||||
// NOTE: Only KQL supports nested
|
||||
describe('kuery', () => {
|
||||
test('it returns formatted query when one item in nested entry', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'included',
|
||||
value: 'value-1',
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ nestedField:"value-1" }');
|
||||
});
|
||||
|
||||
test('it returns formatted query when entry item is "exists"', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'included' }],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ nestedField:* }');
|
||||
});
|
||||
|
||||
test('it returns formatted query when entry item is "exists" and operator is "excluded"', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [{ ...getEntryExistsMock(), field: 'nestedField', operator: 'excluded' }],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ not nestedField:* }');
|
||||
});
|
||||
|
||||
test('it returns formatted query when entry item is "match_any"', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchAnyMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'included',
|
||||
value: ['value1', 'value2'],
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ nestedField:("value1" or "value2") }');
|
||||
});
|
||||
|
||||
test('it returns formatted query when entry item is "match_any" and operator is "excluded"', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchAnyMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'excluded',
|
||||
value: ['value1', 'value2'],
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ not nestedField:("value1" or "value2") }');
|
||||
});
|
||||
|
||||
test('it returns formatted query when multiple items in nested entry', () => {
|
||||
const entry: EntryNested = {
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'included',
|
||||
value: 'value-1',
|
||||
},
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedFieldB',
|
||||
operator: 'included',
|
||||
value: 'value-2',
|
||||
},
|
||||
],
|
||||
};
|
||||
const result = buildNested({ entry, language: 'kuery' });
|
||||
|
||||
expect(result).toEqual('parent:{ nestedField:"value-1" and nestedFieldB:"value-2" }');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildEntry', () => {
|
||||
describe('kuery', () => {
|
||||
test('it returns formatted wildcard string when "type" is "exists"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryExistsMock(), operator: 'included' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(result).toEqual('host.name:*');
|
||||
});
|
||||
|
||||
test('it returns formatted string when "type" is "match"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryMatchMock(), operator: 'included' },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(result).toEqual('host.name:"some host name"');
|
||||
});
|
||||
|
||||
test('it returns formatted string when "type" is "match_any"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
|
||||
language: 'kuery',
|
||||
});
|
||||
expect(result).toEqual('host.name:("some host name" or "auditd")');
|
||||
});
|
||||
});
|
||||
|
||||
describe('lucene', () => {
|
||||
test('it returns formatted wildcard string when "type" is "exists"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryExistsMock(), operator: 'included' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(result).toEqual('_exists_host.name');
|
||||
});
|
||||
|
||||
test('it returns formatted string when "type" is "match"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryMatchMock(), operator: 'included' },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(result).toEqual('host.name:"some host name"');
|
||||
});
|
||||
|
||||
test('it returns formatted string when "type" is "match_any"', () => {
|
||||
const result = buildEntry({
|
||||
entry: { ...getEntryMatchAnyMock(), value: ['some host name', 'auditd'] },
|
||||
language: 'lucene',
|
||||
});
|
||||
expect(result).toEqual('host.name:("some host name" OR "auditd")');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildExceptionItem', () => {
|
||||
test('it returns empty string if empty lists array passed in', () => {
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries: [],
|
||||
});
|
||||
|
||||
expect(query).toEqual('');
|
||||
});
|
||||
|
||||
test('it returns expected query when more than one item in exception item', () => {
|
||||
const payload: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'value-3' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries: payload,
|
||||
});
|
||||
const expectedQuery = 'b:("some host name") and not c:"value-3"';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when exception item includes nested value', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'included',
|
||||
value: 'value-3',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:("some host name") and parent:{ nestedField:"value-3" }';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when exception item includes multiple items and nested "and" values', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'included',
|
||||
value: 'value-3',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ...getEntryExistsMock(), field: 'd' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:("some host name") and parent:{ nestedField:"value-3" } and d:*';
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when language is "lucene"', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{
|
||||
...getEntryMatchMock(),
|
||||
field: 'nestedField',
|
||||
operator: 'excluded',
|
||||
value: 'value-3',
|
||||
},
|
||||
],
|
||||
},
|
||||
{ ...getEntryExistsMock(), field: 'e', operator: 'excluded' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'lucene',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery =
|
||||
'b:("some host name") AND parent:{ NOT nestedField:"value-3" } AND NOT _exists_e';
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
describe('exists', () => {
|
||||
test('it returns expected query when list includes single list item with operator of "included"', () => {
|
||||
const entries: EntriesArray = [{ ...getEntryExistsMock(), field: 'b' }];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:*';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes single list item with operator of "excluded"', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryExistsMock(), field: 'b', operator: 'excluded' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:*';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when exception item includes entry item with "and" values', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryExistsMock(), field: 'b', operator: 'excluded' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'value-1' },
|
||||
],
|
||||
},
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:* and parent:{ c:"value-1" }';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes multiple items', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryExistsMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'value-1' },
|
||||
{ ...getEntryMatchMock(), field: 'd', value: 'value-2' },
|
||||
],
|
||||
},
|
||||
{ ...getEntryExistsMock(), field: 'e' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:* and parent:{ not c:"value-1" and d:"value-2" } and e:*';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
});
|
||||
|
||||
describe('match', () => {
|
||||
test('it returns expected query when list includes single list item with operator of "included"', () => {
|
||||
const entries: EntriesArray = [{ ...getEntryMatchMock(), field: 'b', value: 'value' }];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:"value"';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes single list item with operator of "excluded"', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchMock(), field: 'b', operator: 'excluded', value: 'value' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:"value"';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes list item with "and" values', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchMock(), field: 'b', operator: 'excluded', value: 'value' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'valueC' },
|
||||
],
|
||||
},
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:"value" and parent:{ c:"valueC" }';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes multiple items', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchMock(), field: 'b', value: 'value' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
|
||||
{ ...getEntryMatchMock(), field: 'd', operator: 'excluded', value: 'valueD' },
|
||||
],
|
||||
},
|
||||
{ ...getEntryMatchMock(), field: 'e', value: 'valueE' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery =
|
||||
'b:"value" and parent:{ not c:"valueC" and not d:"valueD" } and e:"valueE"';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
});
|
||||
|
||||
describe('match_any', () => {
|
||||
test('it returns expected query when list includes single list item with operator of "included"', () => {
|
||||
const entries: EntriesArray = [{ ...getEntryMatchAnyMock(), field: 'b' }];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:("some host name")';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes single list item with operator of "excluded"', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b', operator: 'excluded' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:("some host name")';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes list item with nested values', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b', operator: 'excluded' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
|
||||
],
|
||||
},
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'not b:("some host name") and parent:{ not c:"valueC" }';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
|
||||
test('it returns expected query when list includes multiple items', () => {
|
||||
const entries: EntriesArray = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{ ...getEntryMatchAnyMock(), field: 'c' },
|
||||
];
|
||||
const query = buildExceptionItem({
|
||||
language: 'kuery',
|
||||
entries,
|
||||
});
|
||||
const expectedQuery = 'b:("some host name") and c:("some host name")';
|
||||
|
||||
expect(query).toEqual(expectedQuery);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildExceptionListQueries', () => {
|
||||
test('it returns empty array if lists is empty array', () => {
|
||||
const query = buildExceptionListQueries({ language: 'kuery', lists: [] });
|
||||
|
||||
expect(query).toEqual([]);
|
||||
});
|
||||
|
||||
test('it returns empty array if lists is undefined', () => {
|
||||
const query = buildExceptionListQueries({ language: 'kuery', lists: undefined });
|
||||
|
||||
expect(query).toEqual([]);
|
||||
});
|
||||
|
||||
test('it returns expected query when lists exist and language is "kuery"', () => {
|
||||
const payload = getExceptionListItemSchemaMock();
|
||||
const payload2 = getExceptionListItemSchemaMock();
|
||||
payload2.entries = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'included', value: 'valueC' },
|
||||
{ ...getEntryMatchMock(), field: 'd', operator: 'included', value: 'valueD' },
|
||||
],
|
||||
},
|
||||
{ ...getEntryMatchAnyMock(), field: 'e', operator: 'excluded' },
|
||||
];
|
||||
const queries = buildExceptionListQueries({
|
||||
language: 'kuery',
|
||||
lists: [payload, payload2],
|
||||
});
|
||||
const expectedQueries = [
|
||||
{
|
||||
query:
|
||||
'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"',
|
||||
language: 'kuery',
|
||||
},
|
||||
{
|
||||
query:
|
||||
'b:("some host name") and parent:{ c:"valueC" and d:"valueD" } and not e:("some host name")',
|
||||
language: 'kuery',
|
||||
},
|
||||
];
|
||||
|
||||
expect(queries).toEqual(expectedQueries);
|
||||
});
|
||||
|
||||
test('it returns expected query when lists exist and language is "lucene"', () => {
|
||||
const payload = getExceptionListItemSchemaMock();
|
||||
payload.entries = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'a' },
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
];
|
||||
const payload2 = getExceptionListItemSchemaMock();
|
||||
payload2.entries = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'c' },
|
||||
{ ...getEntryMatchAnyMock(), field: 'd' },
|
||||
];
|
||||
const queries = buildExceptionListQueries({
|
||||
language: 'lucene',
|
||||
lists: [payload, payload2],
|
||||
});
|
||||
const expectedQueries = [
|
||||
{
|
||||
query: 'a:("some host name") AND b:("some host name")',
|
||||
language: 'lucene',
|
||||
},
|
||||
{
|
||||
query: 'c:("some host name") AND d:("some host name")',
|
||||
language: 'lucene',
|
||||
},
|
||||
];
|
||||
|
||||
expect(queries).toEqual(expectedQueries);
|
||||
});
|
||||
|
||||
test('it builds correct queries for nested excluded fields', () => {
|
||||
const payload = getExceptionListItemSchemaMock();
|
||||
const payload2 = getExceptionListItemSchemaMock();
|
||||
payload2.entries = [
|
||||
{ ...getEntryMatchAnyMock(), field: 'b' },
|
||||
{
|
||||
field: 'parent',
|
||||
type: 'nested',
|
||||
entries: [
|
||||
// TODO: these operators are not being respected. buildNested needs to be updated
|
||||
{ ...getEntryMatchMock(), field: 'c', operator: 'excluded', value: 'valueC' },
|
||||
{ ...getEntryMatchMock(), field: 'd', operator: 'excluded', value: 'valueD' },
|
||||
],
|
||||
},
|
||||
{ ...getEntryMatchAnyMock(), field: 'e' },
|
||||
];
|
||||
const queries = buildExceptionListQueries({
|
||||
language: 'kuery',
|
||||
lists: [payload, payload2],
|
||||
});
|
||||
const expectedQueries = [
|
||||
{
|
||||
query:
|
||||
'some.parentField:{ nested.field:"some value" } and some.not.nested.field:"some value"',
|
||||
language: 'kuery',
|
||||
},
|
||||
{
|
||||
query:
|
||||
'b:("some host name") and parent:{ not c:"valueC" and not d:"valueD" } and e:("some host name")',
|
||||
language: 'kuery',
|
||||
},
|
||||
];
|
||||
|
||||
expect(queries).toEqual(expectedQueries);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,200 +0,0 @@
|
|||
/*
|
||||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
|
||||
* or more contributor license agreements. Licensed under the Elastic License;
|
||||
* you may not use this file except in compliance with the Elastic License.
|
||||
*/
|
||||
import { Query as DataQuery } from '../../../../../src/plugins/data/common';
|
||||
import {
|
||||
Entry,
|
||||
EntryMatch,
|
||||
EntryMatchAny,
|
||||
EntryNested,
|
||||
EntryExists,
|
||||
EntriesArray,
|
||||
Operator,
|
||||
entriesMatchAny,
|
||||
entriesExists,
|
||||
entriesMatch,
|
||||
entriesNested,
|
||||
ExceptionListItemSchema,
|
||||
CreateExceptionListItemSchema,
|
||||
} from '../shared_imports';
|
||||
import { Language } from './schemas/common/schemas';
|
||||
import { hasLargeValueList } from './utils';
|
||||
|
||||
type Operators = 'and' | 'or' | 'not';
|
||||
type LuceneOperators = 'AND' | 'OR' | 'NOT';
|
||||
|
||||
export const getLanguageBooleanOperator = ({
|
||||
language,
|
||||
value,
|
||||
}: {
|
||||
language: Language;
|
||||
value: Operators;
|
||||
}): Operators | LuceneOperators => {
|
||||
switch (language) {
|
||||
case 'lucene':
|
||||
const luceneValues: Record<Operators, LuceneOperators> = { and: 'AND', or: 'OR', not: 'NOT' };
|
||||
|
||||
return luceneValues[value];
|
||||
case 'kuery':
|
||||
return value;
|
||||
default:
|
||||
return value;
|
||||
}
|
||||
};
|
||||
|
||||
export const operatorBuilder = ({
|
||||
operator,
|
||||
language,
|
||||
}: {
|
||||
operator: Operator;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const not = getLanguageBooleanOperator({
|
||||
language,
|
||||
value: 'not',
|
||||
});
|
||||
|
||||
if (operator === 'excluded') {
|
||||
return `${not} `;
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
export const buildExists = ({
|
||||
entry,
|
||||
language,
|
||||
}: {
|
||||
entry: EntryExists;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const { operator, field } = entry;
|
||||
const exceptionOperator = operatorBuilder({ operator, language });
|
||||
|
||||
switch (language) {
|
||||
case 'kuery':
|
||||
return `${exceptionOperator}${field}:*`;
|
||||
case 'lucene':
|
||||
return `${exceptionOperator}_exists_${field}`;
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
export const buildMatch = ({
|
||||
entry,
|
||||
language,
|
||||
}: {
|
||||
entry: EntryMatch;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const { value, operator, field } = entry;
|
||||
const exceptionOperator = operatorBuilder({ operator, language });
|
||||
|
||||
return `${exceptionOperator}${field}:"${value}"`;
|
||||
};
|
||||
|
||||
export const buildMatchAny = ({
|
||||
entry,
|
||||
language,
|
||||
}: {
|
||||
entry: EntryMatchAny;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const { value, operator, field } = entry;
|
||||
|
||||
switch (value.length) {
|
||||
case 0:
|
||||
return '';
|
||||
default:
|
||||
const or = getLanguageBooleanOperator({ language, value: 'or' });
|
||||
const exceptionOperator = operatorBuilder({ operator, language });
|
||||
const matchAnyValues = value.map((v) => `"${v}"`);
|
||||
|
||||
return `${exceptionOperator}${field}:(${matchAnyValues.join(` ${or} `)})`;
|
||||
}
|
||||
};
|
||||
|
||||
export const buildNested = ({
|
||||
entry,
|
||||
language,
|
||||
}: {
|
||||
entry: EntryNested;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const { field, entries: subentries } = entry;
|
||||
const and = getLanguageBooleanOperator({ language, value: 'and' });
|
||||
const values = subentries.map((subentry) => buildEntry({ entry: subentry, language }));
|
||||
|
||||
return `${field}:{ ${values.join(` ${and} `)} }`;
|
||||
};
|
||||
|
||||
export const buildEntry = ({
|
||||
entry,
|
||||
language,
|
||||
}: {
|
||||
entry: Entry | EntryNested;
|
||||
language: Language;
|
||||
}): string => {
|
||||
if (entriesExists.is(entry)) {
|
||||
return buildExists({ entry, language });
|
||||
} else if (entriesMatch.is(entry)) {
|
||||
return buildMatch({ entry, language });
|
||||
} else if (entriesMatchAny.is(entry)) {
|
||||
return buildMatchAny({ entry, language });
|
||||
} else if (entriesNested.is(entry)) {
|
||||
return buildNested({ entry, language });
|
||||
} else {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
|
||||
export const buildExceptionItem = ({
|
||||
entries,
|
||||
language,
|
||||
}: {
|
||||
entries: EntriesArray;
|
||||
language: Language;
|
||||
}): string => {
|
||||
const and = getLanguageBooleanOperator({ language, value: 'and' });
|
||||
const exceptionItemEntries = entries.map((entry) => {
|
||||
return buildEntry({ entry, language });
|
||||
});
|
||||
|
||||
return exceptionItemEntries.join(` ${and} `);
|
||||
};
|
||||
|
||||
export const buildExceptionListQueries = ({
|
||||
language,
|
||||
lists,
|
||||
}: {
|
||||
language: Language;
|
||||
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema> | undefined;
|
||||
}): DataQuery[] => {
|
||||
if (lists == null || (lists != null && lists.length === 0)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const exceptionItems = lists.reduce<string[]>((acc, exceptionItem) => {
|
||||
const { entries } = exceptionItem;
|
||||
|
||||
if (entries != null && entries.length > 0 && !hasLargeValueList(entries)) {
|
||||
return [...acc, buildExceptionItem({ entries, language })];
|
||||
} else {
|
||||
return acc;
|
||||
}
|
||||
}, []);
|
||||
|
||||
if (exceptionItems.length === 0) {
|
||||
return [];
|
||||
} else {
|
||||
return exceptionItems.map((exceptionItem) => {
|
||||
return {
|
||||
query: exceptionItem,
|
||||
language,
|
||||
};
|
||||
});
|
||||
}
|
||||
};
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -7,7 +7,6 @@
|
|||
import {
|
||||
Filter,
|
||||
IIndexPattern,
|
||||
isFilterDisabled,
|
||||
buildEsQuery,
|
||||
EsQueryConfig,
|
||||
} from '../../../../../src/plugins/data/common';
|
||||
|
|
@ -16,7 +15,7 @@ import {
|
|||
CreateExceptionListItemSchema,
|
||||
} from '../../../lists/common/schemas';
|
||||
import { ESBoolQuery } from '../typed_json';
|
||||
import { buildExceptionListQueries } from './build_exceptions_query';
|
||||
import { buildExceptionFilter } from './build_exceptions_filter';
|
||||
import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
|
||||
|
||||
export const getQueryFilter = (
|
||||
|
|
@ -38,32 +37,27 @@ export const getQueryFilter = (
|
|||
ignoreFilterIfFieldNotInIndex: false,
|
||||
dateFormatTZ: 'Zulu',
|
||||
};
|
||||
|
||||
const enabledFilters = ((filters as unknown) as Filter[]).filter((f) => !isFilterDisabled(f));
|
||||
/*
|
||||
* Pinning exceptions to 'kuery' because lucene
|
||||
* does not support nested queries, while our exceptions
|
||||
* UI does, since we can pass both lucene and kql into
|
||||
* buildEsQuery, this allows us to offer nested queries
|
||||
* regardless
|
||||
*/
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists,
|
||||
config,
|
||||
excludeExceptions,
|
||||
chunkSize: 1024,
|
||||
indexPattern,
|
||||
});
|
||||
if (exceptionFilter !== undefined) {
|
||||
enabledFilters.push(exceptionFilter);
|
||||
}
|
||||
const initialQuery = { query, language };
|
||||
const allFilters = getAllFilters((filters as unknown) as Filter[], exceptionFilter);
|
||||
|
||||
return buildEsQuery(indexPattern, initialQuery, enabledFilters, config);
|
||||
return buildEsQuery(indexPattern, initialQuery, allFilters, config);
|
||||
};
|
||||
|
||||
export const getAllFilters = (filters: Filter[], exceptionFilter: Filter | undefined): Filter[] => {
|
||||
if (exceptionFilter != null) {
|
||||
return [...filters, exceptionFilter];
|
||||
} else {
|
||||
return [...filters];
|
||||
}
|
||||
};
|
||||
|
||||
interface EqlSearchRequest {
|
||||
|
|
@ -84,26 +78,14 @@ export const buildEqlSearchRequest = (
|
|||
eventCategoryOverride: string | undefined
|
||||
): EqlSearchRequest => {
|
||||
const timestamp = timestampOverride ?? '@timestamp';
|
||||
const indexPattern: IIndexPattern = {
|
||||
fields: [],
|
||||
title: index.join(),
|
||||
};
|
||||
const config: EsQueryConfig = {
|
||||
allowLeadingWildcards: true,
|
||||
queryStringOptions: { analyze_wildcard: true },
|
||||
ignoreFilterIfFieldNotInIndex: false,
|
||||
dateFormatTZ: 'Zulu',
|
||||
};
|
||||
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
|
||||
// allowing us to make 1024-item chunks of exception list items.
|
||||
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
|
||||
// very conservative value.
|
||||
const exceptionFilter = buildExceptionFilter({
|
||||
lists: exceptionLists,
|
||||
config,
|
||||
excludeExceptions: true,
|
||||
chunkSize: 1024,
|
||||
indexPattern,
|
||||
});
|
||||
const indexString = index.join();
|
||||
const requestFilter: unknown[] = [
|
||||
|
|
@ -148,69 +130,3 @@ export const buildEqlSearchRequest = (
|
|||
return baseRequest;
|
||||
}
|
||||
};
|
||||
|
||||
export const buildExceptionFilter = ({
|
||||
lists,
|
||||
config,
|
||||
excludeExceptions,
|
||||
chunkSize,
|
||||
indexPattern,
|
||||
}: {
|
||||
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
|
||||
config: EsQueryConfig;
|
||||
excludeExceptions: boolean;
|
||||
chunkSize: number;
|
||||
indexPattern?: IIndexPattern;
|
||||
}) => {
|
||||
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
|
||||
if (exceptionQueries.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const exceptionFilter: Filter = {
|
||||
meta: {
|
||||
alias: null,
|
||||
negate: excludeExceptions,
|
||||
disabled: false,
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
should: undefined,
|
||||
},
|
||||
},
|
||||
};
|
||||
if (exceptionQueries.length <= chunkSize) {
|
||||
const query = buildEsQuery(indexPattern, exceptionQueries, [], config);
|
||||
exceptionFilter.query.bool.should = query.bool.filter;
|
||||
} else {
|
||||
const chunkedFilters: Filter[] = [];
|
||||
for (let index = 0; index < exceptionQueries.length; index += chunkSize) {
|
||||
const exceptionQueriesChunk = exceptionQueries.slice(index, index + chunkSize);
|
||||
const esQueryChunk = buildEsQuery(indexPattern, exceptionQueriesChunk, [], config);
|
||||
const filterChunk: Filter = {
|
||||
meta: {
|
||||
alias: null,
|
||||
negate: false,
|
||||
disabled: false,
|
||||
},
|
||||
query: {
|
||||
bool: {
|
||||
should: esQueryChunk.bool.filter,
|
||||
},
|
||||
},
|
||||
};
|
||||
chunkedFilters.push(filterChunk);
|
||||
}
|
||||
// Here we build a query with only the exceptions: it will put them all in the `filter` array
|
||||
// of the resulting object, which would AND the exceptions together. When creating exceptionFilter,
|
||||
// we move the `filter` array to `should` so they are OR'd together instead.
|
||||
// This gets around the problem with buildEsQuery not allowing callers to specify whether queries passed in
|
||||
// should be ANDed or ORed together.
|
||||
exceptionFilter.query.bool.should = buildEsQuery(
|
||||
indexPattern,
|
||||
[],
|
||||
chunkedFilters,
|
||||
config
|
||||
).bool.filter;
|
||||
}
|
||||
return exceptionFilter;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -55,3 +55,21 @@ export interface EqlSearchResponse<T> {
|
|||
events?: Array<BaseHit<T>>;
|
||||
};
|
||||
}
|
||||
|
||||
export interface BooleanFilter {
|
||||
bool: {
|
||||
must?: unknown | unknown[];
|
||||
must_not?: unknown | unknown[];
|
||||
should?: unknown[];
|
||||
filter?: unknown | unknown[];
|
||||
minimum_should_match?: number;
|
||||
};
|
||||
}
|
||||
|
||||
export interface NestedFilter {
|
||||
nested: {
|
||||
path: string;
|
||||
query: unknown | unknown[];
|
||||
score_mode: string;
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,7 +100,7 @@ describe('Exception viewer helpers', () => {
|
|||
value: undefined,
|
||||
},
|
||||
{
|
||||
fieldName: 'host.name',
|
||||
fieldName: 'parent.field',
|
||||
isNested: false,
|
||||
operator: undefined,
|
||||
value: undefined,
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
import { RequestParams } from '@elastic/elasticsearch';
|
||||
|
||||
import { buildExceptionFilter } from '../../../common/detection_engine/build_exceptions_filter';
|
||||
import { ExceptionListItemSchema } from '../../../../lists/common';
|
||||
import { buildExceptionFilter } from '../../../common/detection_engine/get_query_filter';
|
||||
import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server';
|
||||
import { SearchResponse } from '../types';
|
||||
|
||||
|
|
@ -54,12 +54,6 @@ export const getAnomalies = async (
|
|||
],
|
||||
must_not: buildExceptionFilter({
|
||||
lists: params.exceptionItems,
|
||||
config: {
|
||||
allowLeadingWildcards: true,
|
||||
queryStringOptions: { analyze_wildcard: true },
|
||||
ignoreFilterIfFieldNotInIndex: false,
|
||||
dateFormatTZ: 'Zulu',
|
||||
},
|
||||
excludeExceptions: true,
|
||||
chunkSize: 1024,
|
||||
})?.query,
|
||||
|
|
|
|||
Loading…
Reference in a new issue