maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution][Exceptions] Implement exceptions for ML rules (#84006)

Browse source 
* Implement exceptions for ML rules

* Remove unused import

* Better implicit types

* Retrieve ML rule index pattern for exception field suggestions and autocomplete

* Add ML job logic to edit exception modal

* Remove unnecessary logic change

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>
This commit is contained in:
Marshall Main 2020-12-02 17:14:19 -05:00 committed by GitHub
parent 4f3d72b413
commit d47c70cd53
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 552 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
x-pack/plugins/security_solution/common/detection_engine/get_query_filter.test.ts
View file

@ -7,6 +7,7 @@
import { getQueryFilter, buildExceptionFilter, buildEqlSearchRequest } from './get_query_filter';
import { Filter, EsQueryConfig } from 'src/plugins/data/public';
import { getExceptionListItemSchemaMock } from '../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { ExceptionListItemSchema } from '../shared_imports';
describe('get_filter', () => {
describe('getQueryFilter', () => {
@ -919,19 +920,27 @@ describe('get_filter', () => {
dateFormatTZ: 'Zulu',
};
test('it should build a filter without chunking exception items', () => {
const exceptionFilter = buildExceptionFilter(
[
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
{ language: 'kuery', query: 'user.name: name' },
const exceptionItem1: ExceptionListItemSchema = {
...getExceptionListItemSchemaMock(),
entries: [
{ field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
{ field: 'some.field', operator: 'included', type: 'match', value: 'value' },
],
{
};
const exceptionItem2: ExceptionListItemSchema = {
...getExceptionListItemSchemaMock(),
entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
};
const exceptionFilter = buildExceptionFilter({
lists: [exceptionItem1, exceptionItem2],
config,
excludeExceptions: true,
chunkSize: 2,
indexPattern: {
fields: [],
title: 'auditbeat-*',
},
config,
true,
2
);
});
expect(exceptionFilter).toEqual({
meta: {
alias: null,
@ -949,7 +958,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'host.name': 'linux',
},
},
@ -961,7 +970,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'some.field': 'value',
},
},
@ -976,7 +985,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'user.name': 'name',
},
},
@ -990,20 +999,31 @@ describe('get_filter', () => {
});
test('it should properly chunk exception items', () => {
const exceptionFilter = buildExceptionFilter(
[
{ language: 'kuery', query: 'host.name: linux and some.field: value' },
{ language: 'kuery', query: 'user.name: name' },
{ language: 'kuery', query: 'file.path: /safe/path' },
const exceptionItem1: ExceptionListItemSchema = {
...getExceptionListItemSchemaMock(),
entries: [
{ field: 'host.name', operator: 'included', type: 'match', value: 'linux' },
{ field: 'some.field', operator: 'included', type: 'match', value: 'value' },
],
{
};
const exceptionItem2: ExceptionListItemSchema = {
...getExceptionListItemSchemaMock(),
entries: [{ field: 'user.name', operator: 'included', type: 'match', value: 'name' }],
};
const exceptionItem3: ExceptionListItemSchema = {
...getExceptionListItemSchemaMock(),
entries: [{ field: 'file.path', operator: 'included', type: 'match', value: '/safe/path' }],
};
const exceptionFilter = buildExceptionFilter({
lists: [exceptionItem1, exceptionItem2, exceptionItem3],
config,
excludeExceptions: true,
chunkSize: 2,
indexPattern: {
fields: [],
title: 'auditbeat-*',
},
config,
true,
2
);
});
expect(exceptionFilter).toEqual({
meta: {
alias: null,
@ -1024,7 +1044,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'host.name': 'linux',
},
},
@ -1036,7 +1056,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'some.field': 'value',
},
},
@ -1051,7 +1071,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'user.name': 'name',
},
},
@ -1069,7 +1089,7 @@ describe('get_filter', () => {
minimum_should_match: 1,
should: [
{
match: {
match_phrase: {
'file.path': '/safe/path',
},
},

79
x-pack/plugins/security_solution/common/detection_engine/get_query_filter.ts
View file

@ -6,7 +6,6 @@
import {
Filter,
Query,
IIndexPattern,
isFilterDisabled,
buildEsQuery,
@ -18,15 +17,10 @@ import {
} from '../../../lists/common/schemas';
import { ESBoolQuery } from '../typed_json';
import { buildExceptionListQueries } from './build_exceptions_query';
import {
Query as QueryString,
Language,
Index,
TimestampOverrideOrUndefined,
} from './schemas/common/schemas';
import { Query, Language, Index, TimestampOverrideOrUndefined } from './schemas/common/schemas';
export const getQueryFilter = (
query: QueryString,
query: Query,
language: Language,
filters: Array<Partial<Filter>>,
index: Index,
@ -53,19 +47,18 @@ export const getQueryFilter = (
* buildEsQuery, this allows us to offer nested queries
* regardless
*/
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
if (exceptionQueries.length > 0) {
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
// allowing us to make 1024-item chunks of exception list items.
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
// very conservative value.
const exceptionFilter = buildExceptionFilter(
exceptionQueries,
indexPattern,
config,
excludeExceptions,
1024
);
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
// allowing us to make 1024-item chunks of exception list items.
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
// very conservative value.
const exceptionFilter = buildExceptionFilter({
lists,
config,
excludeExceptions,
chunkSize: 1024,
indexPattern,
});
if (exceptionFilter !== undefined) {
enabledFilters.push(exceptionFilter);
}
const initialQuery = { query, language };
@ -101,15 +94,17 @@ export const buildEqlSearchRequest = (
ignoreFilterIfFieldNotInIndex: false,
dateFormatTZ: 'Zulu',
};
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists: exceptionLists });
let exceptionFilter: Filter | undefined;
if (exceptionQueries.length > 0) {
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
// allowing us to make 1024-item chunks of exception list items.
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
// very conservative value.
exceptionFilter = buildExceptionFilter(exceptionQueries, indexPattern, config, true, 1024);
}
// Assume that `indices.query.bool.max_clause_count` is at least 1024 (the default value),
// allowing us to make 1024-item chunks of exception list items.
// Discussion at https://issues.apache.org/jira/browse/LUCENE-4835 indicates that 1024 is a
// very conservative value.
const exceptionFilter = buildExceptionFilter({
lists: exceptionLists,
config,
excludeExceptions: true,
chunkSize: 1024,
indexPattern,
});
const indexString = index.join();
const requestFilter: unknown[] = [
{
@ -154,13 +149,23 @@ export const buildEqlSearchRequest = (
}
};
export const buildExceptionFilter = (
exceptionQueries: Query[],
indexPattern: IIndexPattern,
config: EsQueryConfig,
excludeExceptions: boolean,
chunkSize: number
) => {
export const buildExceptionFilter = ({
lists,
config,
excludeExceptions,
chunkSize,
indexPattern,
}: {
lists: Array<ExceptionListItemSchema | CreateExceptionListItemSchema>;
config: EsQueryConfig;
excludeExceptions: boolean;
chunkSize: number;
indexPattern?: IIndexPattern;
}) => {
const exceptionQueries = buildExceptionListQueries({ language: 'kuery', lists });
if (exceptionQueries.length === 0) {
return undefined;
}
const exceptionFilter: Filter = {
meta: {
alias: null,

23
x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx
View file

@ -4,6 +4,8 @@
* you may not use this file except in compliance with the Elastic License.
*/
/* eslint complexity: ["error", 30]*/
import React, { memo, useEffect, useState, useCallback, useMemo } from 'react';
import styled, { css } from 'styled-components';
import {
@ -53,6 +55,7 @@ import {
import { ErrorInfo, ErrorCallout } from '../error_callout';
import { ExceptionsBuilderExceptionItem } from '../types';
import { useFetchIndex } from '../../../containers/source';
import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs';
export interface AddExceptionModalProps {
ruleName: string;
@ -108,7 +111,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({
const { http } = useKibana().services;
const [errorsExist, setErrorExists] = useState(false);
const [comment, setComment] = useState('');
const { rule: maybeRule } = useRuleAsync(ruleId);
const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId);
const [shouldCloseAlert, setShouldCloseAlert] = useState(false);
const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false);
const [shouldDisableBulkClose, setShouldDisableBulkClose] = useState(false);
@ -124,8 +127,22 @@ export const AddExceptionModal = memo(function AddExceptionModal({
const [isSignalIndexPatternLoading, { indexPatterns: signalIndexPatterns }] = useFetchIndex(
memoSignalIndexName
);
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices);
const memoMlJobIds = useMemo(
() => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []),
[maybeRule]
);
const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds);
const memoRuleIndices = useMemo(() => {
if (jobs.length > 0) {
return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : [];
} else {
return ruleIndices;
}
}, [jobs, ruleIndices]);
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices);
const onError = useCallback(
(error: Error): void => {
addError(error, { title: i18n.ADD_EXCEPTION_ERROR });
@ -364,6 +381,8 @@ export const AddExceptionModal = memo(function AddExceptionModal({
!isSignalIndexPatternLoading &&
!isLoadingExceptionList &&
!isIndexPatternLoading &&
!isRuleLoading &&
!mlJobLoading &&
ruleExceptionList && (
<>
<ModalBodySection className="builder-section">

149
x-pack/plugins/security_solution/public/common/components/exceptions/edit_exception_modal/index.tsx
View file

@ -47,6 +47,7 @@ import {
} from '../helpers';
import { Loader } from '../../loader';
import { ErrorInfo, ErrorCallout } from '../error_callout';
import { useGetInstalledJob } from '../../ml/hooks/use_get_jobs';
interface EditExceptionModalProps {
ruleName: string;
@ -100,7 +101,7 @@ export const EditExceptionModal = memo(function EditExceptionModal({
const { http } = useKibana().services;
const [comment, setComment] = useState('');
const [errorsExist, setErrorExists] = useState(false);
const { rule: maybeRule } = useRuleAsync(ruleId);
const { rule: maybeRule, loading: isRuleLoading } = useRuleAsync(ruleId);
const [updateError, setUpdateError] = useState<ErrorInfo | null>(null);
const [hasVersionConflict, setHasVersionConflict] = useState(false);
const [shouldBulkCloseAlert, setShouldBulkCloseAlert] = useState(false);
@ -117,7 +118,21 @@ export const EditExceptionModal = memo(function EditExceptionModal({
memoSignalIndexName
);
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(ruleIndices);
const memoMlJobIds = useMemo(
() => (maybeRule?.machine_learning_job_id != null ? [maybeRule.machine_learning_job_id] : []),
[maybeRule]
);
const { loading: mlJobLoading, jobs } = useGetInstalledJob(memoMlJobIds);
const memoRuleIndices = useMemo(() => {
if (jobs.length > 0) {
return jobs[0].results_index_name ? [`.ml-anomalies-${jobs[0].results_index_name}`] : [];
} else {
return ruleIndices;
}
}, [jobs, ruleIndices]);
const [isIndexPatternLoading, { indexPatterns }] = useFetchIndex(memoRuleIndices);
const handleExceptionUpdateError = useCallback(
(error: Error, statusCode: number | null, message: string | null) => {
@ -280,69 +295,75 @@ export const EditExceptionModal = memo(function EditExceptionModal({
{(addExceptionIsLoading || isIndexPatternLoading || isSignalIndexLoading) && (
<Loader data-test-subj="loadingEditExceptionModal" size="xl" />
)}
{!isSignalIndexLoading && !addExceptionIsLoading && !isIndexPatternLoading && (
<>
<ModalBodySection className="builder-section">
{isRuleEQLSequenceStatement && (
<>
<EuiCallOut
data-test-subj="eql-sequence-callout"
title={i18n.EDIT_EXCEPTION_SEQUENCE_WARNING}
/>
<EuiSpacer />
</>
)}
<EuiText>{i18n.EXCEPTION_BUILDER_INFO}</EuiText>
<EuiSpacer />
<ExceptionBuilderComponent
exceptionListItems={[exceptionItem]}
listType={exceptionListType}
listId={exceptionItem.list_id}
listNamespaceType={exceptionItem.namespace_type}
ruleName={ruleName}
isOrDisabled
isAndDisabled={false}
isNestedDisabled={false}
data-test-subj="edit-exception-modal-builder"
id-aria="edit-exception-modal-builder"
onChange={handleBuilderOnChange}
indexPatterns={indexPatterns}
ruleType={maybeRule?.type}
/>
<EuiSpacer />
<AddExceptionComments
exceptionItemComments={exceptionItem.comments}
newCommentValue={comment}
newCommentOnChange={onCommentChange}
/>
</ModalBodySection>
<EuiHorizontalRule />
<ModalBodySection>
<EuiFormRow fullWidth>
<EuiCheckbox
data-test-subj="close-alert-on-add-edit-exception-checkbox"
id="close-alert-on-add-edit-exception-checkbox"
label={
shouldDisableBulkClose ? i18n.BULK_CLOSE_LABEL_DISABLED : i18n.BULK_CLOSE_LABEL
}
checked={shouldBulkCloseAlert}
onChange={onBulkCloseAlertCheckboxChange}
disabled={shouldDisableBulkClose}
{!isSignalIndexLoading &&
!addExceptionIsLoading &&
!isIndexPatternLoading &&
!isRuleLoading &&
!mlJobLoading && (
<>
<ModalBodySection className="builder-section">
{isRuleEQLSequenceStatement && (
<>
<EuiCallOut
data-test-subj="eql-sequence-callout"
title={i18n.EDIT_EXCEPTION_SEQUENCE_WARNING}
/>
<EuiSpacer />
</>
)}
<EuiText>{i18n.EXCEPTION_BUILDER_INFO}</EuiText>
<EuiSpacer />
<ExceptionBuilderComponent
exceptionListItems={[exceptionItem]}
listType={exceptionListType}
listId={exceptionItem.list_id}
listNamespaceType={exceptionItem.namespace_type}
ruleName={ruleName}
isOrDisabled
isAndDisabled={false}
isNestedDisabled={false}
data-test-subj="edit-exception-modal-builder"
id-aria="edit-exception-modal-builder"
onChange={handleBuilderOnChange}
indexPatterns={indexPatterns}
ruleType={maybeRule?.type}
/>
</EuiFormRow>
{exceptionListType === 'endpoint' && (
<>
<EuiSpacer />
<EuiText data-test-subj="edit-exception-endpoint-text" color="subdued" size="s">
{i18n.ENDPOINT_QUARANTINE_TEXT}
</EuiText>
</>
)}
</ModalBodySection>
</>
)}
<EuiSpacer />
<AddExceptionComments
exceptionItemComments={exceptionItem.comments}
newCommentValue={comment}
newCommentOnChange={onCommentChange}
/>
</ModalBodySection>
<EuiHorizontalRule />
<ModalBodySection>
<EuiFormRow fullWidth>
<EuiCheckbox
data-test-subj="close-alert-on-add-edit-exception-checkbox"
id="close-alert-on-add-edit-exception-checkbox"
label={
shouldDisableBulkClose
? i18n.BULK_CLOSE_LABEL_DISABLED
: i18n.BULK_CLOSE_LABEL
}
checked={shouldBulkCloseAlert}
onChange={onBulkCloseAlertCheckboxChange}
disabled={shouldDisableBulkClose}
/>
</EuiFormRow>
{exceptionListType === 'endpoint' && (
<>
<EuiSpacer />
<EuiText data-test-subj="edit-exception-endpoint-text" color="subdued" size="s">
{i18n.ENDPOINT_QUARANTINE_TEXT}
</EuiText>
</>
)}
</ModalBodySection>
</>
)}
{updateError != null && (
<ModalBodySection>
<ErrorCallout

35
x-pack/plugins/security_solution/public/common/components/ml/api/get_jobs.ts Normal file
View file

@ -0,0 +1,35 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { CombinedJobWithStats } from '../../../../../../ml/common/types/anomaly_detection_jobs';
import { HttpSetup } from '../../../../../../../../src/core/public';
export interface GetJobsArgs {
http: HttpSetup;
jobIds: string[];
signal: AbortSignal;
}
/**
* Fetches details for a set of ML jobs
*
* @param http HTTP Service
* @param jobIds Array of job IDs to filter against
* @param signal to cancel request
*
* @throws An error if response is not OK
*/
export const getJobs = async ({
http,
jobIds,
signal,
}: GetJobsArgs): Promise<CombinedJobWithStats[]> =>
http.fetch<CombinedJobWithStats[]>('/api/ml/jobs/jobs', {
method: 'POST',
body: JSON.stringify({ jobIds }),
asSystemRequest: true,
signal,
});

59
x-pack/plugins/security_solution/public/common/components/ml/hooks/use_get_jobs.ts Normal file
View file

@ -0,0 +1,59 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
import { useEffect, useState } from 'react';
import { useAsync, withOptionalSignal } from '../../../../shared_imports';
import { getJobs } from '../api/get_jobs';
import { CombinedJobWithStats } from '../../../../../../ml/common/types/anomaly_detection_jobs';
import { hasMlUserPermissions } from '../../../../../common/machine_learning/has_ml_user_permissions';
import { hasMlLicense } from '../../../../../common/machine_learning/has_ml_license';
import { useAppToasts } from '../../../hooks/use_app_toasts';
import { useHttp } from '../../../lib/kibana';
import { useMlCapabilities } from './use_ml_capabilities';
import * as i18n from '../translations';
const _getJobs = withOptionalSignal(getJobs);
export const useGetJobs = () => useAsync(_getJobs);
export interface UseGetInstalledJobReturn {
loading: boolean;
jobs: CombinedJobWithStats[];
isMlUser: boolean;
isLicensed: boolean;
}
export const useGetInstalledJob = (jobIds: string[]): UseGetInstalledJobReturn => {
const [jobs, setJobs] = useState<CombinedJobWithStats[]>([]);
const { addError } = useAppToasts();
const mlCapabilities = useMlCapabilities();
const http = useHttp();
const { error, loading, result, start } = useGetJobs();
const isMlUser = hasMlUserPermissions(mlCapabilities);
const isLicensed = hasMlLicense(mlCapabilities);
useEffect(() => {
if (isMlUser && isLicensed && jobIds.length > 0) {
start({ http, jobIds });
}
}, [http, isMlUser, isLicensed, start, jobIds]);
useEffect(() => {
if (result) {
setJobs(result);
}
}, [result]);
useEffect(() => {
if (error) {
addError(error, { title: i18n.SIEM_JOB_FETCH_FAILURE });
}
}, [addError, error]);
return { isLicensed, isMlUser, jobs, loading };
};

19
x-pack/plugins/security_solution/public/detections/components/alerts_table/timeline_actions/alert_context_menu.tsx
View file

@ -21,7 +21,6 @@ import { TimelineId } from '../../../../../common/types/timeline';
import { DEFAULT_INDEX_PATTERN } from '../../../../../common/constants';
import { Status, Type } from '../../../../../common/detection_engine/schemas/common/schemas';
import { isThresholdRule } from '../../../../../common/detection_engine/utils';
import { isMlRule } from '../../../../../common/machine_learning/helpers';
import { timelineActions } from '../../../../timelines/store/timeline';
import { EventsTd, EventsTdContent } from '../../../../timelines/components/timeline/styles';
import { DEFAULT_ICON_BUTTON_WIDTH } from '../../../../timelines/components/timeline/helpers';
@ -75,11 +74,17 @@ const AlertContextMenuComponent: React.FC<AlertContextMenuProps> = ({
'',
[ecsRowData]
);
const ruleIndices = useMemo(
(): string[] =>
(ecsRowData.signal?.rule && ecsRowData.signal.rule.index) ?? DEFAULT_INDEX_PATTERN,
[ecsRowData]
);
const ruleIndices = useMemo((): string[] => {
if (
ecsRowData.signal?.rule &&
ecsRowData.signal.rule.index &&
ecsRowData.signal.rule.index.length > 0
) {
return ecsRowData.signal.rule.index;
} else {
return DEFAULT_INDEX_PATTERN;
}
}, [ecsRowData]);
const { addWarning } = useAppToasts();
@ -321,7 +326,7 @@ const AlertContextMenuComponent: React.FC<AlertContextMenuProps> = ({
const areExceptionsAllowed = useMemo((): boolean => {
const ruleTypes = getOr([], 'signal.rule.type', ecsRowData);
const [ruleType] = ruleTypes as Type[];
return !isMlRule(ruleType) && !isThresholdRule(ruleType);
return !isThresholdRule(ruleType);
}, [ecsRowData]);
// eslint-disable-next-line react-hooks/exhaustive-deps

6
x-pack/plugins/security_solution/public/detections/components/rules/step_about_rule/index.tsx
View file

@ -8,7 +8,6 @@ import { EuiAccordion, EuiFlexItem, EuiSpacer, EuiFormRow } from '@elastic/eui';
import React, { FC, memo, useCallback, useEffect, useState } from 'react';
import styled from 'styled-components';
import { isMlRule } from '../../../../../common/machine_learning/helpers';
import { isThresholdRule } from '../../../../../common/detection_engine/utils';
import {
RuleStepProps,
@ -76,10 +75,7 @@ const StepAboutRuleComponent: FC<StepAboutRuleProps> = ({
const [severityValue, setSeverityValue] = useState<string>(initialState.severity.value);
const [indexPatternLoading, { indexPatterns }] = useFetchIndex(defineRuleData?.index ?? []);
const canUseExceptions =
defineRuleData?.ruleType &&
!isMlRule(defineRuleData.ruleType) &&
!isThresholdRule(defineRuleData.ruleType);
const canUseExceptions = defineRuleData?.ruleType && !isThresholdRule(defineRuleData.ruleType);
const { form } = useForm<AboutStepRule>({
defaultValue: initialState,

3
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/details/index.tsx
View file

@ -80,7 +80,6 @@ import { DEFAULT_INDEX_PATTERN } from '../../../../../../common/constants';
import { useFullScreen } from '../../../../../common/containers/use_full_screen';
import { Display } from '../../../../../hosts/pages/display';
import { ExceptionListTypeEnum, ExceptionListIdentifiers } from '../../../../../shared_imports';
import { isMlRule } from '../../../../../../common/machine_learning/helpers';
import { isThresholdRule } from '../../../../../../common/detection_engine/utils';
import { useRuleAsync } from '../../../../containers/detection_engine/rules/use_rule_async';
import { showGlobalFilters } from '../../../../../timelines/components/timeline/helpers';
@ -104,7 +103,7 @@ enum RuleDetailTabs {
}
const getRuleDetailsTabs = (rule: Rule | null) => {
const canUseExceptions = rule && !isMlRule(rule.type) && !isThresholdRule(rule.type);
const canUseExceptions = rule && !isThresholdRule(rule.type);
return [
{
id: RuleDetailTabs.alerts,

8
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -150,8 +150,8 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig
export const sampleDocWithSortId = (
someUuid: string = sampleIdGuid,
ip?: string,
destIp?: string
ip?: string | string[],
destIp?: string | string[]
): SignalSourceHit => ({
_index: 'myFakeSignalIndex',
_type: 'doc',
@ -502,8 +502,8 @@ export const repeatedSearchResultsWithSortId = (
total: number,
pageSize: number,
guids: string[],
ips?: string[],
destIps?: string[]
ips?: Array<string | string[]>,
destIps?: Array<string | string[]>
): SignalSearchResponse => ({
took: 10,
timed_out: false,

2
x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts
View file

@ -6,7 +6,6 @@
import { flow, omit } from 'lodash/fp';
import set from 'set-value';
import { SearchResponse } from 'elasticsearch';
import { Logger } from '../../../../../../../src/core/server';
import { AlertServices } from '../../../../../alerts/server';
@ -15,6 +14,7 @@ import { RuleTypeParams, RefreshTypes } from '../types';
import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create';
import { AnomalyResults, Anomaly } from '../../machine_learning';
import { BuildRuleMessage } from './rule_messages';
import { SearchResponse } from '../../types';
interface BulkCreateMlSignalsParams {
actions: RuleAlertAction[];

162
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts
View file

@ -400,6 +400,87 @@ describe('filterEventsAgainstList', () => {
'9.9.9.9',
]).toEqual(ipVals);
});
it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
{
field: 'destination.ip',
operator: 'included',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
]);
// this call represents an exception list with a value list containing ['4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '4.4.4.4' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(
3,
3,
someGuids.slice(0, 3),
[
['1.1.1.1', '1.1.1.1'],
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
],
[
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
['3.3.3.3', '4.4.4.4'],
]
),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
]);
expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
]);
expect(res.hits.hits.length).toEqual(2);
// @ts-expect-error
const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip);
expect([
['1.1.1.1', '1.1.1.1'],
['1.1.1.1', '2.2.2.2'],
]).toEqual(sourceIpVals);
// @ts-expect-error
const destIpVals = res.hits.hits.map((item) => item._source.destination.ip);
expect([
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
]).toEqual(destIpVals);
});
});
describe('operator type is excluded', () => {
it('should respond with empty list if no items match value list', async () => {
@ -463,5 +544,86 @@ describe('filterEventsAgainstList', () => {
);
expect(res.hits.hits.length).toEqual(2);
});
it('should respond with less items in the list given one exception item with two entries of type list and array of values in document', async () => {
const exceptionItem = getExceptionListItemSchemaMock();
exceptionItem.entries = [
{
field: 'source.ip',
operator: 'excluded',
type: 'list',
list: {
id: 'ci-badguys.txt',
type: 'ip',
},
},
{
field: 'destination.ip',
operator: 'excluded',
type: 'list',
list: {
id: 'ci-badguys-again.txt',
type: 'ip',
},
},
];
// this call represents an exception list with a value list containing ['2.2.2.2']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '2.2.2.2' },
]);
// this call represents an exception list with a value list containing ['4.4.4.4']
(listClient.getListItemByValues as jest.Mock).mockResolvedValueOnce([
{ ...getListItemResponseMock(), value: '4.4.4.4' },
]);
const res = await filterEventsAgainstList({
logger: mockLogger,
listClient,
exceptionsList: [exceptionItem],
eventSearchResult: repeatedSearchResultsWithSortId(
3,
3,
someGuids.slice(0, 3),
[
['1.1.1.1', '1.1.1.1'],
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
],
[
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
['3.3.3.3', '4.4.4.4'],
]
),
buildRuleMessage,
});
expect(listClient.getListItemByValues as jest.Mock).toHaveBeenCalledTimes(2);
expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].value).toEqual([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
]);
expect((listClient.getListItemByValues as jest.Mock).mock.calls[1][0].value).toEqual([
'1.1.1.1',
'2.2.2.2',
'3.3.3.3',
'4.4.4.4',
]);
expect(res.hits.hits.length).toEqual(2);
// @ts-expect-error
const sourceIpVals = res.hits.hits.map((item) => item._source.source.ip);
expect([
['1.1.1.1', '2.2.2.2'],
['2.2.2.2', '3.3.3.3'],
]).toEqual(sourceIpVals);
// @ts-expect-error
const destIpVals = res.hits.hits.map((item) => item._source.destination.ip);
expect([
['2.2.2.2', '3.3.3.3'],
['3.3.3.3', '4.4.4.4'],
]).toEqual(destIpVals);
});
});
});

71
x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts
View file

@ -7,7 +7,6 @@ import { get } from 'lodash/fp';
import { Logger } from 'src/core/server';
import { ListClient } from '../../../../../lists/server';
import { SignalSearchResponse } from './types';
import { BuildRuleMessage } from './rule_messages';
import {
EntryList,
@ -17,16 +16,23 @@ import {
} from '../../../../../lists/common/schemas';
import { hasLargeValueList } from '../../../../common/detection_engine/utils';
import { SearchTypes } from '../../../../common/detection_engine/types';
import { SearchResponse } from '../../types';
interface FilterEventsAgainstList {
listClient: ListClient;
exceptionsList: ExceptionListItemSchema[];
logger: Logger;
eventSearchResult: SignalSearchResponse;
buildRuleMessage: BuildRuleMessage;
}
// narrow unioned type to be single
const isStringableType = (val: SearchTypes): val is string | number | boolean =>
['string', 'number', 'boolean'].includes(typeof val);
export const createSetToFilterAgainst = async ({
const isStringableArray = (val: SearchTypes): val is Array<string | number | boolean> => {
if (!Array.isArray(val)) {
return false;
}
// TS does not allow .every to be called on val as-is, even though every type in the union
// is an array. https://github.com/microsoft/TypeScript/issues/36390
// @ts-expect-error
return val.every((subVal) => isStringableType(subVal));
};
export const createSetToFilterAgainst = async <T>({
events,
field,
listId,
@ -35,7 +41,7 @@ export const createSetToFilterAgainst = async ({
logger,
buildRuleMessage,
}: {
events: SignalSearchResponse['hits']['hits'];
events: SearchResponse<T>['hits']['hits'];
field: string;
listId: string;
listType: Type;
@ -43,13 +49,14 @@ export const createSetToFilterAgainst = async ({
logger: Logger;
buildRuleMessage: BuildRuleMessage;
}): Promise<Set<SearchTypes>> => {
// narrow unioned type to be single
const isStringableType = (val: SearchTypes) =>
['string', 'number', 'boolean'].includes(typeof val);
const valuesFromSearchResultField = events.reduce((acc, searchResultItem) => {
const valueField = get(field, searchResultItem._source);
if (valueField != null && isStringableType(valueField)) {
acc.add(valueField.toString());
if (valueField != null) {
if (isStringableType(valueField)) {
acc.add(valueField.toString());
} else if (isStringableArray(valueField)) {
valueField.forEach((subVal) => acc.add(subVal.toString()));
}
}
return acc;
}, new Set<string>());
@ -71,13 +78,19 @@ export const createSetToFilterAgainst = async ({
return matchedListItemsSet;
};
export const filterEventsAgainstList = async ({
export const filterEventsAgainstList = async <T>({
listClient,
exceptionsList,
logger,
eventSearchResult,
buildRuleMessage,
}: FilterEventsAgainstList): Promise<SignalSearchResponse> => {
}: {
listClient: ListClient;
exceptionsList: ExceptionListItemSchema[];
logger: Logger;
eventSearchResult: SearchResponse<T>;
buildRuleMessage: BuildRuleMessage;
}): Promise<SearchResponse<T>> => {
try {
if (exceptionsList == null || exceptionsList.length === 0) {
logger.debug(buildRuleMessage('about to return original search result'));
@ -108,9 +121,9 @@ export const filterEventsAgainstList = async ({
});
// now that we have all the exception items which are value lists (whether single entry or have multiple entries)
const res = await valueListExceptionItems.reduce<Promise<SignalSearchResponse['hits']['hits']>>(
const res = await valueListExceptionItems.reduce<Promise<SearchResponse<T>['hits']['hits']>>(
async (
filteredAccum: Promise<SignalSearchResponse['hits']['hits']>,
filteredAccum: Promise<SearchResponse<T>['hits']['hits']>,
exceptionItem: ExceptionListItemSchema
) => {
// 1. acquire the values from the specified fields to check
@ -152,15 +165,23 @@ export const filterEventsAgainstList = async ({
const vals = fieldAndSetTuples.map((tuple) => {
const eventItem = get(tuple.field, item._source);
if (tuple.operator === 'included') {
// only create a signal if the event is not in the value list
// only create a signal if the field value is not in the value list
if (eventItem != null) {
return !tuple.matchedSet.has(eventItem);
if (isStringableType(eventItem)) {
return !tuple.matchedSet.has(eventItem);
} else if (isStringableArray(eventItem)) {
return !eventItem.some((val) => tuple.matchedSet.has(val));
}
}
return true;
} else if (tuple.operator === 'excluded') {
// only create a signal if the event is in the value list
// only create a signal if the field value is in the value list
if (eventItem != null) {
return tuple.matchedSet.has(eventItem);
if (isStringableType(eventItem)) {
return tuple.matchedSet.has(eventItem);
} else if (isStringableArray(eventItem)) {
return eventItem.some((val) => tuple.matchedSet.has(val));
}
}
return true;
}
@ -175,10 +196,10 @@ export const filterEventsAgainstList = async ({
const toReturn = filteredEvents;
return toReturn;
},
Promise.resolve<SignalSearchResponse['hits']['hits']>(eventSearchResult.hits.hits)
Promise.resolve<SearchResponse<T>['hits']['hits']>(eventSearchResult.hits.hits)
);
const toReturn: SignalSearchResponse = {
const toReturn: SearchResponse<T> = {
took: eventSearchResult.took,
timed_out: eventSearchResult.timed_out,
_shards: eventSearchResult._shards,

4
x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_ml_signals.ts
View file

@ -5,6 +5,7 @@
*/
import dateMath from '@elastic/datemath';
import { ExceptionListItemSchema } from '../../../../../lists/common';
import { KibanaRequest, SavedObjectsClientContract } from '../../../../../../../src/core/server';
import { MlPluginSetup } from '../../../../../ml/server';
@ -18,6 +19,7 @@ export const findMlSignals = async ({
anomalyThreshold,
from,
to,
exceptionItems,
}: {
ml: MlPluginSetup;
request: KibanaRequest;
@ -26,6 +28,7 @@ export const findMlSignals = async ({
anomalyThreshold: number;
from: string;
to: string;
exceptionItems: ExceptionListItemSchema[];
}): Promise<AnomalyResults> => {
const { mlAnomalySearch } = ml.mlSystemProvider(request, savedObjectsClient);
const params = {
@ -33,6 +36,7 @@ export const findMlSignals = async ({
threshold: anomalyThreshold,
earliestMs: dateMath.parse(from)?.valueOf() ?? 0,
latestMs: dateMath.parse(to)?.valueOf() ?? 0,
exceptionItems,
};
return getAnomalies(params, mlAnomalySearch);
};

21
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -66,6 +66,7 @@ import { buildSignalFromEvent, buildSignalGroupFromSequence } from './build_bulk
import { createThreatSignals } from './threat_mapping/create_threat_signals';
import { getIndexVersion } from '../routes/index/get_index_version';
import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template';
import { filterEventsAgainstList } from './filter_events_with_list';
export const signalRulesAlertType = ({
logger,
@ -242,9 +243,18 @@ export const signalRulesAlertType = ({
anomalyThreshold,
from,
to,
exceptionItems: exceptionItems ?? [],
});
const anomalyCount = anomalyResults.hits.hits.length;
const filteredAnomalyResults = await filterEventsAgainstList({
listClient,
exceptionsList: exceptionItems ?? [],
logger,
eventSearchResult: anomalyResults,
buildRuleMessage,
});
const anomalyCount = filteredAnomalyResults.hits.hits.length;
if (anomalyCount) {
logger.info(buildRuleMessage(`Found ${anomalyCount} signals from ML anomalies.`));
}
@ -257,7 +267,7 @@ export const signalRulesAlertType = ({
} = await bulkCreateMlSignals({
actions,
throttle,
someResult: anomalyResults,
someResult: filteredAnomalyResults,
ruleParams: params,
services,
logger,
@ -276,15 +286,16 @@ export const signalRulesAlertType = ({
});
// The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] }
const shardFailures =
(anomalyResults._shards as typeof anomalyResults._shards & { failures: [] }).failures ??
[];
(filteredAnomalyResults._shards as typeof filteredAnomalyResults._shards & {
failures: [];
}).failures ?? [];
const searchErrors = createErrorsFromShard({
errors: shardFailures,
});
result = mergeReturns([
result,
createSearchAfterReturnType({
success: success && anomalyResults._shards.failed === 0,
success: success && filteredAnomalyResults._shards.failed === 0,
errors: [...errors, ...searchErrors],
createdSignalsCount: createdItemsCount,
bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [],

2
x-pack/plugins/security_solution/server/lib/machine_learning/index.test.ts
View file

@ -4,6 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { getExceptionListItemSchemaMock } from '../../../../lists/common/schemas/response/exception_list_item_schema.mock';
import { getAnomalies, AnomaliesSearchParams } from '.';
const getFiltersFromMock = (mock: jest.Mock) => {
@ -23,6 +24,7 @@ describe('getAnomalies', () => {
threshold: 5,
earliestMs: 1588517231429,
latestMs: 1588617231429,
exceptionItems: [getExceptionListItemSchemaMock(), getExceptionListItemSchemaMock()],
};
});

16
x-pack/plugins/security_solution/server/lib/machine_learning/index.ts
View file

@ -4,10 +4,12 @@
* you may not use this file except in compliance with the Elastic License.
*/
import { SearchResponse } from 'elasticsearch';
import { RequestParams } from '@elastic/elasticsearch';
import { ExceptionListItemSchema } from '../../../../lists/common';
import { buildExceptionFilter } from '../../../common/detection_engine/get_query_filter';
import { AnomalyRecordDoc as Anomaly } from '../../../../ml/server';
import { SearchResponse } from '../types';
export { Anomaly };
export type AnomalyResults = SearchResponse<Anomaly>;
@ -21,6 +23,7 @@ export interface AnomaliesSearchParams {
threshold: number;
earliestMs: number;
latestMs: number;
exceptionItems: ExceptionListItemSchema[];
maxRecords?: number;
}
@ -49,6 +52,17 @@ export const getAnomalies = async (
},
},
],
must_not: buildExceptionFilter({
lists: params.exceptionItems,
config: {
allowLeadingWildcards: true,
queryStringOptions: { analyze_wildcard: true },
ignoreFilterIfFieldNotInIndex: false,
dateFormatTZ: 'Zulu',
},
excludeExceptions: true,
chunkSize: 1024,
})?.query,
},
},
sort: [{ record_score: { order: 'desc' } }],