maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[Security Solution][Detections][7.12] Critical Threshold Rule Fixes (#92667)

Browse source 
* Threshold cardinality validation

* Remove comments

* Fix legacy threshold signal dupe mitigation

* Add find_threshold_signals tests

* remove comment

* bug fixes

* Fix edit form value initialization for cardinality_value

* Fix test

* Type and test fixes

* Tests/types

* Reenable threshold cypress test

* Schema fixes

* Types and tests, normalize threshold field util

* Continue cleaning up types

* Some more pre-7.12 tests

* Limit cardinality_field to length 1 for now

* Cardinality to array

* Cardinality to array

* Tests/types

* cardinality can be null

* Handle empty threshold field in bulk_create_threshold_signals

* Remove cardinality_field, cardinality_value
This commit is contained in:
Madison Caldwell 2021-03-01 17:10:22 -05:00 committed by GitHub
parent cd38671565
commit cb053f4672
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 1398 additions and 358 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

48
x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
View file

@ -465,26 +465,56 @@ export type Threats = t.TypeOf<typeof threats>;
export const threatsOrUndefined = t.union([threats, t.undefined]);
export type ThreatsOrUndefined = t.TypeOf<typeof threatsOrUndefined>;
export const thresholdField = t.exact(
t.type({
field: t.union([t.string, t.array(t.string)]), // Covers pre- and post-7.12
value: PositiveIntegerGreaterThanZero,
})
);
export type ThresholdField = t.TypeOf<typeof thresholdField>;
export const thresholdFieldNormalized = t.exact(
t.type({
field: t.array(t.string),
value: PositiveIntegerGreaterThanZero,
})
);
export type ThresholdFieldNormalized = t.TypeOf<typeof thresholdFieldNormalized>;
export const thresholdCardinalityField = t.exact(
t.type({
field: t.string,
value: PositiveInteger,
})
);
export type ThresholdCardinalityField = t.TypeOf<typeof thresholdCardinalityField>;
export const threshold = t.intersection([
t.exact(
t.type({
field: t.union([t.string, t.array(t.string)]),
value: PositiveIntegerGreaterThanZero,
})
),
thresholdField,
t.exact(
t.partial({
cardinality_field: t.union([t.string, t.array(t.string), t.undefined, t.null]),
cardinality_value: t.union([PositiveInteger, t.undefined, t.null]), // TODO: cardinality_value should be set if cardinality_field is set
cardinality: t.union([t.array(thresholdCardinalityField), t.null]),
})
),
]);
// TODO: codec to transform threshold field string to string[] ?
export type Threshold = t.TypeOf<typeof threshold>;
export const thresholdOrUndefined = t.union([threshold, t.undefined]);
export type ThresholdOrUndefined = t.TypeOf<typeof thresholdOrUndefined>;
export const thresholdNormalized = t.intersection([
thresholdFieldNormalized,
t.exact(
t.partial({
cardinality: t.union([t.array(thresholdCardinalityField), t.null]),
})
),
]);
export type ThresholdNormalized = t.TypeOf<typeof thresholdNormalized>;
export const thresholdNormalizedOrUndefined = t.union([thresholdNormalized, t.undefined]);
export type ThresholdNormalizedOrUndefined = t.TypeOf<typeof thresholdNormalizedOrUndefined>;
export const created_at = IsoDateString;
export const updated_at = IsoDateString;
export const updated_by = t.string;

26
x-pack/plugins/security_solution/common/detection_engine/utils.test.ts
View file

@ -5,7 +5,13 @@
* 2.0.
*/
import { hasEqlSequenceQuery, hasLargeValueList, hasNestedEntry, isThreatMatchRule } from './utils';
import {
hasEqlSequenceQuery,
hasLargeValueList,
hasNestedEntry,
isThreatMatchRule,
normalizeThresholdField,
} from './utils';
import { EntriesArray } from '../shared_imports';
describe('#hasLargeValueList', () => {
@ -151,3 +157,21 @@ describe('#hasEqlSequenceQuery', () => {
});
});
});
describe('normalizeThresholdField', () => {
it('converts a string to a string array', () => {
expect(normalizeThresholdField('host.name')).toEqual(['host.name']);
});
it('returns a string array when a string array is passed in', () => {
expect(normalizeThresholdField(['host.name'])).toEqual(['host.name']);
});
it('converts undefined to an empty array', () => {
expect(normalizeThresholdField(undefined)).toEqual([]);
});
it('converts null to an empty array', () => {
expect(normalizeThresholdField(null)).toEqual([]);
});
it('converts an empty string to an empty array', () => {
expect(normalizeThresholdField('')).toEqual([]);
});
});

12
x-pack/plugins/security_solution/common/detection_engine/utils.ts
View file

@ -5,6 +5,8 @@
* 2.0.
*/
import { isEmpty } from 'lodash';
import {
CreateExceptionListItemSchema,
EntriesArray,
@ -42,3 +44,13 @@ export const isQueryRule = (ruleType: Type | undefined): boolean =>
ruleType === 'query' || ruleType === 'saved_query';
export const isThreatMatchRule = (ruleType: Type | undefined): boolean =>
ruleType === 'threat_match';
export const normalizeThresholdField = (
thresholdField: string | string[] | null | undefined
): string[] => {
return Array.isArray(thresholdField)
? thresholdField
: isEmpty(thresholdField)
? []
: [thresholdField!];
};

6
x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts
View file

@ -40,8 +40,10 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions {
| {
field: string | string[] | undefined;
value: number;
cardinality_field?: string | undefined;
cardinality_value?: number | undefined;
cardinality?: {
field: string[];
value: number;
};
}
| undefined;
inspect?: Maybe<Inspect>;

187
x-pack/plugins/security_solution/cypress/integration/detection_rules/threshold_rule.spec.ts
View file

@ -79,103 +79,100 @@ import { loginAndWaitForPageWithoutDateRange } from '../../tasks/login';
import { DETECTIONS_URL } from '../../urls/navigation';
// Skipped until post-FF for 7.12
describe.skip('Threshold Rules', () => {
describe('Detection rules, threshold', () => {
const expectedUrls = newThresholdRule.referenceUrls.join('');
const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join('');
const expectedTags = newThresholdRule.tags.join('');
const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre);
describe('Detection rules, threshold', () => {
const expectedUrls = newThresholdRule.referenceUrls.join('');
const expectedFalsePositives = newThresholdRule.falsePositivesExamples.join('');
const expectedTags = newThresholdRule.tags.join('');
const expectedMitre = formatMitreAttackDescription(newThresholdRule.mitre);
const rule = { ...newThresholdRule };
const rule = { ...newThresholdRule };
beforeEach(() => {
cleanKibana();
createTimeline(newThresholdRule.timeline).then((response) => {
rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId;
});
});
it('Creates and activates a new threshold rule', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
waitForAlertsIndexToBeCreated();
goToManageAlertsDetectionRules();
waitForRulesTableToBeLoaded();
goToCreateNewRule();
selectThresholdRuleType();
fillDefineThresholdRuleAndContinue(rule);
fillAboutRuleAndContinue(rule);
fillScheduleRuleAndContinue(rule);
createAndActivateRule();
cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)');
changeRowsPerPageTo300();
const expectedNumberOfRules = 1;
cy.get(RULES_TABLE).then(($table) => {
cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules);
});
filterByCustomRules();
cy.get(RULES_TABLE).then(($table) => {
cy.wrap($table.find(RULES_ROW).length).should('eql', 1);
});
cy.get(RULE_NAME).should('have.text', rule.name);
cy.get(RISK_SCORE).should('have.text', rule.riskScore);
cy.get(SEVERITY).should('have.text', rule.severity);
cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true');
goToRuleDetails();
cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`);
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description);
cy.get(ABOUT_DETAILS).within(() => {
getDetails(SEVERITY_DETAILS).should('have.text', rule.severity);
getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore);
getDetails(REFERENCE_URLS_DETAILS).should((details) => {
expect(removeExternalLinkText(details.text())).equal(expectedUrls);
});
getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives);
getDetails(MITRE_ATTACK_DETAILS).should((mitre) => {
expect(removeExternalLinkText(mitre.text())).equal(expectedMitre);
});
getDetails(TAGS_DETAILS).should('have.text', expectedTags);
});
cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN);
cy.get(DEFINITION_DETAILS).within(() => {
getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join(''));
getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery);
getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold');
getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None');
getDetails(THRESHOLD_DETAILS).should(
'have.text',
`Results aggregated by ${rule.thresholdField} >= ${rule.threshold}`
);
});
cy.get(SCHEDULE_DETAILS).within(() => {
getDetails(RUNS_EVERY_DETAILS).should(
'have.text',
`${rule.runsEvery.interval}${rule.runsEvery.type}`
);
getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should(
'have.text',
`${rule.lookBack.interval}${rule.lookBack.type}`
);
});
waitForTheRuleToBeExecuted();
waitForAlertsToPopulate();
cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100));
cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name);
cy.get(ALERT_RULE_VERSION).first().should('have.text', '1');
cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold');
cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase());
cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore);
beforeEach(() => {
cleanKibana();
createTimeline(newThresholdRule.timeline).then((response) => {
rule.timeline.id = response.body.data.persistTimeline.timeline.savedObjectId;
});
});
it('Creates and activates a new threshold rule', () => {
loginAndWaitForPageWithoutDateRange(DETECTIONS_URL);
waitForAlertsPanelToBeLoaded();
waitForAlertsIndexToBeCreated();
goToManageAlertsDetectionRules();
waitForRulesTableToBeLoaded();
goToCreateNewRule();
selectThresholdRuleType();
fillDefineThresholdRuleAndContinue(rule);
fillAboutRuleAndContinue(rule);
fillScheduleRuleAndContinue(rule);
createAndActivateRule();
cy.get(CUSTOM_RULES_BTN).should('have.text', 'Custom rules (1)');
changeRowsPerPageTo300();
const expectedNumberOfRules = 1;
cy.get(RULES_TABLE).then(($table) => {
cy.wrap($table.find(RULES_ROW).length).should('eql', expectedNumberOfRules);
});
filterByCustomRules();
cy.get(RULES_TABLE).then(($table) => {
cy.wrap($table.find(RULES_ROW).length).should('eql', 1);
});
cy.get(RULE_NAME).should('have.text', rule.name);
cy.get(RISK_SCORE).should('have.text', rule.riskScore);
cy.get(SEVERITY).should('have.text', rule.severity);
cy.get(RULE_SWITCH).should('have.attr', 'aria-checked', 'true');
goToRuleDetails();
cy.get(RULE_NAME_HEADER).should('have.text', `${rule.name}`);
cy.get(ABOUT_RULE_DESCRIPTION).should('have.text', rule.description);
cy.get(ABOUT_DETAILS).within(() => {
getDetails(SEVERITY_DETAILS).should('have.text', rule.severity);
getDetails(RISK_SCORE_DETAILS).should('have.text', rule.riskScore);
getDetails(REFERENCE_URLS_DETAILS).should((details) => {
expect(removeExternalLinkText(details.text())).equal(expectedUrls);
});
getDetails(FALSE_POSITIVES_DETAILS).should('have.text', expectedFalsePositives);
getDetails(MITRE_ATTACK_DETAILS).should((mitre) => {
expect(removeExternalLinkText(mitre.text())).equal(expectedMitre);
});
getDetails(TAGS_DETAILS).should('have.text', expectedTags);
});
cy.get(INVESTIGATION_NOTES_TOGGLE).click({ force: true });
cy.get(ABOUT_INVESTIGATION_NOTES).should('have.text', INVESTIGATION_NOTES_MARKDOWN);
cy.get(DEFINITION_DETAILS).within(() => {
getDetails(INDEX_PATTERNS_DETAILS).should('have.text', indexPatterns.join(''));
getDetails(CUSTOM_QUERY_DETAILS).should('have.text', rule.customQuery);
getDetails(RULE_TYPE_DETAILS).should('have.text', 'Threshold');
getDetails(TIMELINE_TEMPLATE_DETAILS).should('have.text', 'None');
getDetails(THRESHOLD_DETAILS).should(
'have.text',
`Results aggregated by ${rule.thresholdField} >= ${rule.threshold}`
);
});
cy.get(SCHEDULE_DETAILS).within(() => {
getDetails(RUNS_EVERY_DETAILS).should(
'have.text',
`${rule.runsEvery.interval}${rule.runsEvery.type}`
);
getDetails(ADDITIONAL_LOOK_BACK_DETAILS).should(
'have.text',
`${rule.lookBack.interval}${rule.lookBack.type}`
);
});
waitForTheRuleToBeExecuted();
waitForAlertsToPopulate();
cy.get(NUMBER_OF_ALERTS).should(($count) => expect(+$count.text()).to.be.lt(100));
cy.get(ALERT_RULE_NAME).first().should('have.text', rule.name);
cy.get(ALERT_RULE_VERSION).first().should('have.text', '1');
cy.get(ALERT_RULE_METHOD).first().should('have.text', 'threshold');
cy.get(ALERT_RULE_SEVERITY).first().should('have.text', rule.severity.toLowerCase());
cy.get(ALERT_RULE_RISK_SCORE).first().should('have.text', rule.riskScore);
});
});

6
x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts
View file

@ -78,8 +78,10 @@ export interface MatrixHistogramQueryProps {
| {
field: string | string[] | undefined;
value: number;
cardinality_field?: string | undefined;
cardinality_value?: number | undefined;
cardinality?: {
field: string[];
value: number;
};
}
| undefined;
skip?: boolean;

24
x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.test.tsx
View file

@ -296,8 +296,10 @@ describe('PreviewQuery', () => {
threshold={{
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
}}
isDisabled={false}
/>
@ -343,8 +345,10 @@ describe('PreviewQuery', () => {
threshold={{
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
}}
isDisabled={false}
/>
@ -387,8 +391,10 @@ describe('PreviewQuery', () => {
threshold={{
field: undefined,
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
}}
isDisabled={false}
/>
@ -419,8 +425,10 @@ describe('PreviewQuery', () => {
threshold={{
field: ' ',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
}}
isDisabled={false}
/>

6
x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx
View file

@ -60,8 +60,10 @@ export type Threshold =
| {
field: string | string[] | undefined;
value: number;
cardinality_field: string | undefined;
cardinality_value: number | undefined;
cardinality?: {
field: string[];
value: number;
};
}
| undefined;

36
x-pack/plugins/security_solution/public/detections/components/rules/query_preview/reducer.test.ts
View file

@ -337,8 +337,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'threshold',
});
@ -355,8 +357,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: undefined,
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'threshold',
});
@ -373,8 +377,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: ' ',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'threshold',
});
@ -391,8 +397,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'eql',
});
@ -408,8 +416,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'query',
});
@ -425,8 +435,10 @@ describe('queryPreviewReducer', () => {
threshold: {
field: 'agent.hostname',
value: 200,
cardinality_field: 'user.name',
cardinality_value: 2,
cardinality: {
field: ['user.name'],
value: 2,
},
},
ruleType: 'saved_query',
});

35
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/index.tsx
View file

@ -82,8 +82,10 @@ const stepDefineDefaultValue: DefineStepRule = {
threshold: {
field: [],
value: '200',
cardinality_field: [],
cardinality_value: '2',
cardinality: {
field: [],
value: '',
},
},
timeline: {
id: null,
@ -154,15 +156,15 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
threatIndex: formThreatIndex,
'threshold.field': formThresholdField,
'threshold.value': formThresholdValue,
'threshold.cardinality_field': formThresholdCardinalityField,
'threshold.cardinality_value': formThresholdCardinalityValue,
'threshold.cardinality.field': formThresholdCardinalityField,
'threshold.cardinality.value': formThresholdCardinalityValue,
},
] = useFormData<
DefineStepRule & {
'threshold.field': string[] | undefined;
'threshold.value': number | undefined;
'threshold.cardinality_field': string[] | undefined;
'threshold.cardinality_value': number | undefined;
'threshold.cardinality.field': string[] | undefined;
'threshold.cardinality.value': number | undefined;
}
>({
form,
@ -172,8 +174,8 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
'queryBar',
'threshold.field',
'threshold.value',
'threshold.cardinality_field',
'threshold.cardinality_value',
'threshold.cardinality.field',
'threshold.cardinality.value',
'threatIndex',
],
});
@ -289,15 +291,14 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
}, []);
const thresholdFormValue = useMemo((): Threshold | undefined => {
return formThresholdValue != null &&
formThresholdField != null &&
formThresholdCardinalityField != null &&
formThresholdCardinalityValue != null
return formThresholdValue != null
? {
field: formThresholdField[0],
field: formThresholdField ?? [],
value: formThresholdValue,
cardinality_field: formThresholdCardinalityField[0],
cardinality_value: formThresholdCardinalityValue,
cardinality: {
field: formThresholdCardinalityField ?? [],
value: formThresholdCardinalityValue ?? 0, // FIXME
},
}
: undefined;
}, [
@ -460,10 +461,10 @@ const StepDefineRuleComponent: FC<StepDefineRuleProps> = ({
path: 'threshold.value',
},
thresholdCardinalityField: {
path: 'threshold.cardinality_field',
path: 'threshold.cardinality.field',
},
thresholdCardinalityValue: {
path: 'threshold.cardinality_value',
path: 'threshold.cardinality.value',
},
}}
>

123
x-pack/plugins/security_solution/public/detections/components/rules/step_define_rule/schema.tsx
View file

@ -239,52 +239,85 @@ export const schema: FormSchema<DefineStepRule> = {
},
],
},
cardinality_field: {
type: FIELD_TYPES.COMBO_BOX,
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel',
{
defaultMessage: 'Count',
}
),
helpText: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText',
{
defaultMessage: 'Select a field to check cardinality',
}
),
},
cardinality_value: {
type: FIELD_TYPES.NUMBER,
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel',
{
defaultMessage: 'Unique values',
}
),
validations: [
{
validator: (
...args: Parameters<ValidationFunc>
): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
const [{ formData }] = args;
const needsValidation = isThresholdRule(formData.ruleType);
if (!needsValidation) {
return;
}
return fieldValidators.numberGreaterThanField({
than: 1,
message: i18n.translate(
'xpack.securitySolution.detectionEngine.validations.thresholdValueFieldData.numberGreaterThanOrEqualOneErrorMessage',
{
defaultMessage: 'Value must be greater than or equal to one.',
}
),
allowEquality: true,
})(...args);
cardinality: {
field: {
defaultValue: [],
fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'],
type: FIELD_TYPES.COMBO_BOX,
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityFieldLabel',
{
defaultMessage: 'Count',
}
),
validations: [
{
validator: (
...args: Parameters<ValidationFunc>
): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
const [{ formData }] = args;
const needsValidation = isThresholdRule(formData.ruleType);
if (!needsValidation) {
return;
}
if (
isEmpty(formData['threshold.cardinality.field']) &&
!isEmpty(formData['threshold.cardinality.value'])
) {
return fieldValidators.emptyField(
i18n.translate(
'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityFieldFieldData.thresholdCardinalityFieldNotSuppliedMessage',
{
defaultMessage: 'A Cardinality Field is required.',
}
)
)(...args);
}
},
},
},
],
],
helpText: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdFieldCardinalityFieldHelpText',
{
defaultMessage: 'Select a field to check cardinality',
}
),
},
value: {
fieldsToValidateOnChange: ['threshold.cardinality.field', 'threshold.cardinality.value'],
type: FIELD_TYPES.NUMBER,
label: i18n.translate(
'xpack.securitySolution.detectionEngine.createRule.stepAboutRule.fieldThresholdCardinalityValueFieldLabel',
{
defaultMessage: 'Unique values',
}
),
validations: [
{
validator: (
...args: Parameters<ValidationFunc>
): ReturnType<ValidationFunc<{}, ERROR_CODE>> | undefined => {
const [{ formData }] = args;
const needsValidation = isThresholdRule(formData.ruleType);
if (!needsValidation) {
return;
}
if (!isEmpty(formData['threshold.cardinality.field'])) {
return fieldValidators.numberGreaterThanField({
than: 1,
message: i18n.translate(
'xpack.securitySolution.detectionEngine.validations.thresholdCardinalityValueFieldData.numberGreaterThanOrEqualOneErrorMessage',
{
defaultMessage: 'Value must be greater than or equal to one.',
}
),
allowEquality: true,
})(...args);
}
},
},
],
},
},
},
threatIndex: {

6
x-pack/plugins/security_solution/public/detections/components/rules/threshold_input/index.tsx
View file

@ -19,8 +19,10 @@ const FIELD_COMBO_BOX_WIDTH = 410;
export interface FieldValueThreshold {
field: string[];
value: string;
cardinality_field: string[];
cardinality_value: string;
cardinality?: {
field: string[];
value: string;
};
}
interface ThresholdInputProps {

16
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/__mocks__/mock.ts
View file

@ -143,8 +143,12 @@ export const mockRuleWithEverything = (id: string): Rule => ({
threshold: {
field: ['host.name'],
value: 50,
cardinality_field: ['process.name'],
cardinality_value: 2,
cardinality: [
{
field: 'process.name',
value: 2,
},
],
},
throttle: 'no_actions',
timestamp_override: 'event.ingested',
@ -192,10 +196,12 @@ export const mockDefineStepRule = (): DefineStepRule => ({
},
threatIndex: [],
threshold: {
field: [''],
field: [],
value: '100',
cardinality_field: [''],
cardinality_value: '2',
cardinality: {
field: ['process.name'],
value: '2',
},
},
});

12
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/create/helpers.ts
View file

@ -221,8 +221,16 @@ export const formatDefineStepData = (defineStepData: DefineStepRule): DefineStep
threshold: {
field: ruleFields.threshold?.field ?? [],
value: parseInt(ruleFields.threshold?.value, 10) ?? 0,
cardinality_field: ruleFields.threshold.cardinality_field[0] ?? '',
cardinality_value: parseInt(ruleFields.threshold?.cardinality_value, 10) ?? 0,
cardinality:
!isEmpty(ruleFields.threshold.cardinality?.field) &&
ruleFields.threshold.cardinality?.value != null
? [
{
field: ruleFields.threshold.cardinality.field[0],
value: parseInt(ruleFields.threshold.cardinality.value, 10),
},
]
: [],
},
}),
}

10
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.test.tsx
View file

@ -84,8 +84,10 @@ describe('rule helpers', () => {
threshold: {
field: ['host.name'],
value: '50',
cardinality_field: ['process.name'],
cardinality_value: '2',
cardinality: {
field: ['process.name'],
value: '2',
},
},
threatIndex: [],
threatMapping: [],
@ -215,8 +217,6 @@ describe('rule helpers', () => {
threshold: {
field: [],
value: '100',
cardinality_field: [],
cardinality_value: '0',
},
threatIndex: [],
threatMapping: [],
@ -259,8 +259,6 @@ describe('rule helpers', () => {
threshold: {
field: [],
value: '100',
cardinality_field: [],
cardinality_value: '0',
},
threatIndex: [],
threatMapping: [],

21
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/helpers.tsx
View file

@ -13,6 +13,7 @@ import { useLocation } from 'react-router-dom';
import styled from 'styled-components';
import { EuiFlexItem } from '@elastic/eui';
import { ActionVariables } from '../../../../../../triggers_actions_ui/public';
import { normalizeThresholdField } from '../../../../../common/detection_engine/utils';
import { RuleAlertAction } from '../../../../../common/detection_engine/types';
import { assertUnreachable } from '../../../../../common/utility_types';
import { transformRuleToAlertAction } from '../../../../../common/detection_engine/transform_actions';
@ -99,18 +100,16 @@ export const getDefineStepsData = (rule: Rule): DefineStepRule => ({
title: rule.timeline_title ?? null,
},
threshold: {
field: rule.threshold?.field
? Array.isArray(rule.threshold.field)
? rule.threshold.field
: [rule.threshold.field]
: [],
field: normalizeThresholdField(rule.threshold?.field),
value: `${rule.threshold?.value || 100}`,
cardinality_field: Array.isArray(rule.threshold?.cardinality_field)
? rule.threshold!.cardinality_field
: rule.threshold?.cardinality_field != null
? [rule.threshold!.cardinality_field]
: [],
cardinality_value: `${rule.threshold?.cardinality_value ?? 0}`,
...(rule.threshold?.cardinality?.length
? {
cardinality: {
field: [`${rule.threshold.cardinality[0].field}`],
value: `${rule.threshold.cardinality[0].value}`,
},
}
: {}),
},
});

6
x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/types.ts
View file

@ -160,8 +160,10 @@ export interface DefineStepRuleJson {
threshold?: {
field: string[];
value: number;
cardinality_field: string;
cardinality_value: number;
cardinality: Array<{
field: string;
value: number;
}>;
};
threat_query?: string;
threat_mapping?: ThreatMapping;

6
x-pack/plugins/security_solution/server/lib/detection_engine/schemas/rule_converters.ts
View file

@ -7,6 +7,7 @@
import uuid from 'uuid';
import { InternalRuleCreate, InternalRuleResponse, TypeSpecificRuleParams } from './rule_schemas';
import { normalizeThresholdField } from '../../../../common/detection_engine/utils';
import { assertUnreachable } from '../../../../common/utility_types';
import {
CreateRulesSchema,
@ -207,7 +208,10 @@ export const typeSpecificCamelToSnake = (params: TypeSpecificRuleParams): Respon
query: params.query,
filters: params.filters,
saved_id: params.savedId,
threshold: params.threshold,
threshold: {
...params.threshold,
field: normalizeThresholdField(params.threshold.field),
},
};
}
case 'machine_learning': {

35
x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file

@ -434,8 +434,12 @@ export const sampleThresholdSignalHit = (): SignalHit => ({
threshold: {
field: ['host.name'],
value: 5,
cardinality_field: 'process.name',
cardinality_value: 2,
cardinality: [
{
field: 'process.name',
value: 2,
},
],
},
updated_by: 'elastic_kibana',
tags: ['some fake tag 1', 'some fake tag 2'],
@ -460,6 +464,25 @@ export const sampleThresholdSignalHit = (): SignalHit => ({
},
});
const sampleThresholdHit = sampleThresholdSignalHit();
export const sampleLegacyThresholdSignalHit = (): unknown => ({
...sampleThresholdHit,
signal: {
...sampleThresholdHit.signal,
rule: {
...sampleThresholdHit.signal.rule,
threshold: {
field: 'host.name',
value: 5,
},
},
threshold_result: {
count: 72,
value: 'a hostname',
},
},
});
export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => {
return {
_index: 'myFakeSignalIndex',
@ -468,6 +491,14 @@ export const sampleWrappedThresholdSignalHit = (): WrappedSignalHit => {
};
};
export const sampleWrappedLegacyThresholdSignalHit = (): WrappedSignalHit => {
return {
_index: 'myFakeSignalIndex',
_id: 'adb9d636-fbbe-4962-ac1c-e282f3ec5879',
_source: sampleLegacyThresholdSignalHit() as SignalHit,
};
};
export const sampleBulkCreateDuplicateResult = {
took: 60,
errors: true,

250
x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.test.ts
View file

@ -9,15 +9,172 @@ import { loggingSystemMock } from '../../../../../../../src/core/server/mocks';
import { sampleDocNoSortId, sampleDocSearchResultsNoSortId } from './__mocks__/es_results';
import { transformThresholdResultsToEcs } from './bulk_create_threshold_signals';
import { calculateThresholdSignalUuid } from './utils';
import { Threshold } from '../../../../common/detection_engine/schemas/common/schemas';
import { normalizeThresholdField } from '../../../../common/detection_engine/utils';
import {
Threshold,
ThresholdNormalized,
} from '../../../../common/detection_engine/schemas/common/schemas';
describe('transformThresholdResultsToEcs', () => {
it('should return transformed threshold results', () => {
describe('transformThresholdNormalizedResultsToEcs', () => {
it('should return transformed threshold results for pre-7.12 rules', () => {
const threshold: Threshold = {
field: 'source.ip',
value: 1,
};
const startedAt = new Date('2020-12-17T16:27:00Z');
const transformedResults = transformThresholdResultsToEcs(
{
...sampleDocSearchResultsNoSortId('abcd'),
aggregations: {
'threshold_0:source.ip': {
buckets: [
{
key: '127.0.0.1',
doc_count: 15,
top_threshold_hits: {
hits: {
hits: [sampleDocNoSortId('abcd')],
},
},
},
],
},
},
},
'test',
startedAt,
undefined,
loggingSystemMock.createLogger(),
{
...threshold,
field: normalizeThresholdField(threshold.field),
},
'1234',
undefined
);
const _id = calculateThresholdSignalUuid('1234', startedAt, ['source.ip'], '127.0.0.1');
expect(transformedResults).toEqual({
took: 10,
timed_out: false,
_shards: {
total: 10,
successful: 10,
failed: 0,
skipped: 0,
},
results: {
hits: {
total: 1,
},
},
hits: {
total: 100,
max_score: 100,
hits: [
{
_id,
_index: 'test',
_source: {
'@timestamp': '2020-04-20T21:27:45+0000',
threshold_result: {
terms: [
{
field: 'source.ip',
value: '127.0.0.1',
},
],
cardinality: undefined,
count: 15,
},
},
},
],
},
});
});
it('should return transformed threshold results for pre-7.12 rules without threshold field', () => {
const threshold: Threshold = {
field: '',
value: 1,
};
const startedAt = new Date('2020-12-17T16:27:00Z');
const transformedResults = transformThresholdResultsToEcs(
{
...sampleDocSearchResultsNoSortId('abcd'),
aggregations: {
threshold_0: {
buckets: [
{
key: '',
doc_count: 15,
top_threshold_hits: {
hits: {
hits: [sampleDocNoSortId('abcd')],
},
},
},
],
},
},
},
'test',
startedAt,
undefined,
loggingSystemMock.createLogger(),
{
...threshold,
field: normalizeThresholdField(threshold.field),
},
'1234',
undefined
);
const _id = calculateThresholdSignalUuid('1234', startedAt, [], '');
expect(transformedResults).toEqual({
took: 10,
timed_out: false,
_shards: {
total: 10,
successful: 10,
failed: 0,
skipped: 0,
},
results: {
hits: {
total: 1,
},
},
hits: {
total: 100,
max_score: 100,
hits: [
{
_id,
_index: 'test',
_source: {
'@timestamp': '2020-04-20T21:27:45+0000',
threshold_result: {
terms: [],
cardinality: undefined,
count: 15,
},
},
},
],
},
});
});
it('should return transformed threshold results', () => {
const threshold: ThresholdNormalized = {
field: ['source.ip', 'host.name'],
value: 1,
cardinality_field: 'destination.ip',
cardinality_value: 5,
cardinality: [
{
field: 'destination.ip',
value: 5,
},
],
};
const startedAt = new Date('2020-12-17T16:27:00Z');
const transformedResults = transformThresholdResultsToEcs(
@ -112,4 +269,87 @@ describe('transformThresholdResultsToEcs', () => {
},
});
});
it('should return transformed threshold results without threshold fields', () => {
const threshold: ThresholdNormalized = {
field: [],
value: 1,
cardinality: [
{
field: 'destination.ip',
value: 5,
},
],
};
const startedAt = new Date('2020-12-17T16:27:00Z');
const transformedResults = transformThresholdResultsToEcs(
{
...sampleDocSearchResultsNoSortId('abcd'),
aggregations: {
threshold_0: {
buckets: [
{
key: '',
doc_count: 15,
top_threshold_hits: {
hits: {
hits: [sampleDocNoSortId('abcd')],
},
},
cardinality_count: {
value: 7,
},
},
],
},
},
},
'test',
startedAt,
undefined,
loggingSystemMock.createLogger(),
threshold,
'1234',
undefined
);
const _id = calculateThresholdSignalUuid('1234', startedAt, [], '');
expect(transformedResults).toEqual({
took: 10,
timed_out: false,
_shards: {
total: 10,
successful: 10,
failed: 0,
skipped: 0,
},
results: {
hits: {
total: 1,
},
},
hits: {
total: 100,
max_score: 100,
hits: [
{
_id,
_index: 'test',
_source: {
'@timestamp': '2020-04-20T21:27:45+0000',
threshold_result: {
terms: [],
cardinality: [
{
field: 'destination.ip',
value: 7,
},
],
count: 15,
},
},
},
],
},
});
});
});

83
x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_threshold_signals.ts
View file

@ -5,11 +5,12 @@
* 2.0.
*/
import { get, isEmpty } from 'lodash/fp';
import { get } from 'lodash/fp';
import set from 'set-value';
import { normalizeThresholdField } from '../../../../common/detection_engine/utils';
import {
Threshold,
ThresholdNormalized,
TimestampOverrideOrUndefined,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { Logger } from '../../../../../../../src/core/server';
@ -56,59 +57,19 @@ const getTransformedHits = (
inputIndex: string,
startedAt: Date,
logger: Logger,
threshold: Threshold,
threshold: ThresholdNormalized,
ruleId: string,
filter: unknown,
timestampOverride: TimestampOverrideOrUndefined
) => {
if (isEmpty(threshold.field)) {
const totalResults =
typeof results.hits.total === 'number' ? results.hits.total : results.hits.total.value;
const aggParts = threshold.field.length
? results.aggregations && getThresholdAggregationParts(results.aggregations)
: {
field: null,
index: 0,
name: 'threshold_0',
};
if (totalResults < threshold.value) {
return [];
}
const hit = results.hits.hits[0];
if (hit == null) {
logger.warn(`No hits returned, but totalResults >= threshold.value (${threshold.value})`);
return [];
}
const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields);
if (timestampArray == null) {
return [];
}
const timestamp = timestampArray[0];
if (typeof timestamp !== 'string') {
return [];
}
const source = {
'@timestamp': timestamp,
threshold_result: {
terms: [
{
value: ruleId,
},
],
count: totalResults,
},
};
return [
{
_index: inputIndex,
_id: calculateThresholdSignalUuid(
ruleId,
startedAt,
Array.isArray(threshold.field) ? threshold.field : [threshold.field]
),
_source: source,
},
];
}
const aggParts = results.aggregations && getThresholdAggregationParts(results.aggregations);
if (!aggParts) {
return [];
}
@ -119,7 +80,7 @@ const getTransformedHits = (
const nextLevelIdx = i + 1;
const nextLevelAggParts = getThresholdAggregationParts(bucket, nextLevelIdx);
if (nextLevelAggParts == null) {
throw new Error('Something went horribly wrong');
throw new Error('Unable to parse aggregation.');
}
const nextLevelPath = `['${nextLevelAggParts.name}']['buckets']`;
const nextBuckets = get(nextLevelPath, bucket);
@ -132,7 +93,7 @@ const getTransformedHits = (
value: bucket.key,
},
...val.terms,
],
].filter((term) => term.field != null),
cardinality: val.cardinality,
topThresholdHits: val.topThresholdHits,
docCount: val.docCount,
@ -146,13 +107,11 @@ const getTransformedHits = (
field,
value: bucket.key,
},
],
cardinality: !isEmpty(threshold.cardinality_field)
].filter((term) => term.field != null),
cardinality: threshold.cardinality?.length
? [
{
field: Array.isArray(threshold.cardinality_field)
? threshold.cardinality_field[0]
: threshold.cardinality_field!,
field: threshold.cardinality[0].field,
value: bucket.cardinality_count!.value,
},
]
@ -208,7 +167,7 @@ const getTransformedHits = (
_id: calculateThresholdSignalUuid(
ruleId,
startedAt,
Array.isArray(threshold.field) ? threshold.field : [threshold.field],
threshold.field,
bucket.terms.map((term) => term.value).join(',')
),
_source: source,
@ -226,7 +185,7 @@ export const transformThresholdResultsToEcs = (
startedAt: Date,
filter: unknown,
logger: Logger,
threshold: Threshold,
threshold: ThresholdNormalized,
ruleId: string,
timestampOverride: TimestampOverrideOrUndefined
): SignalSearchResponse => {
@ -259,13 +218,17 @@ export const bulkCreateThresholdSignals = async (
params: BulkCreateThresholdSignalsParams
): Promise<SingleBulkCreateResponse> => {
const thresholdResults = params.someResult;
const threshold = params.ruleParams.threshold!;
const ecsResults = transformThresholdResultsToEcs(
thresholdResults,
params.inputIndexPattern.join(','),
params.startedAt,
params.filter,
params.logger,
params.ruleParams.threshold!,
{
...threshold,
field: normalizeThresholdField(threshold.field),
},
params.ruleParams.ruleId,
params.timestampOverride
);

445
x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.test.ts Normal file
View file

@ -0,0 +1,445 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { getQueryFilter } from '../../../../common/detection_engine/get_query_filter';
import { mockLogger } from './__mocks__/es_results';
import { findThresholdSignals } from './find_threshold_signals';
import { buildRuleMessageFactory } from './rule_messages';
import * as single_search_after from './single_search_after';
const buildRuleMessage = buildRuleMessageFactory({
id: 'fake id',
ruleId: 'fake rule id',
index: 'fakeindex',
name: 'fake name',
});
const queryFilter = getQueryFilter('', 'kuery', [], ['*'], []);
const mockSingleSearchAfter = jest.fn();
describe('findThresholdSignals', () => {
let mockService: AlertServicesMock;
beforeEach(() => {
jest.clearAllMocks();
jest.spyOn(single_search_after, 'singleSearchAfter').mockImplementation(mockSingleSearchAfter);
mockService = alertsMock.createAlertServices();
});
it('should generate a threshold signal for pre-7.12 rules', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: 'host.name',
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
'threshold_0:host.name': {
terms: {
field: 'host.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
})
);
});
it('should generate a signal for pre-7.12 rules with no threshold field', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: '',
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
threshold_0: {
terms: {
script: {
source: '""',
lang: 'painless',
},
min_doc_count: 100,
},
aggs: {
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
})
);
});
it('should generate a threshold signal query when only a value is provided', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: [],
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
threshold_0: {
terms: {
script: {
source: '""',
lang: 'painless',
},
min_doc_count: 100,
},
aggs: {
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
})
);
});
it('should generate a threshold signal query when a field and value are provided', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: ['host.name'],
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
'threshold_0:host.name': {
terms: {
field: 'host.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
})
);
});
it('should generate a threshold signal query when multiple fields and a value are provided', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: ['host.name', 'user.name'],
value: 100,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
'threshold_0:host.name': {
terms: {
field: 'host.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
'threshold_1:user.name': {
terms: {
field: 'user.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
},
},
})
);
});
it('should generate a threshold signal query when multiple fields, a value, and cardinality field/value are provided', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
field: ['host.name', 'user.name'],
value: 100,
cardinality: [
{
field: 'destination.ip',
value: 2,
},
],
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
'threshold_0:host.name': {
terms: {
field: 'host.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
'threshold_1:user.name': {
terms: {
field: 'user.name',
min_doc_count: 100,
size: 10000,
},
aggs: {
cardinality_count: {
cardinality: {
field: 'destination.ip',
},
},
cardinality_check: {
bucket_selector: {
buckets_path: {
cardinalityCount: 'cardinality_count',
},
script: 'params.cardinalityCount >= 2',
},
},
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
},
},
})
);
});
it('should generate a threshold signal query when only a value and a cardinality field/value are provided', async () => {
await findThresholdSignals({
from: 'now-6m',
to: 'now',
inputIndexPattern: ['*'],
services: mockService,
logger: mockLogger,
filter: queryFilter,
threshold: {
cardinality: [
{
field: 'source.ip',
value: 5,
},
],
field: [],
value: 200,
},
buildRuleMessage,
timestampOverride: undefined,
});
expect(mockSingleSearchAfter).toHaveBeenCalledWith(
expect.objectContaining({
aggregations: {
threshold_0: {
terms: {
script: {
source: '""',
lang: 'painless',
},
min_doc_count: 200,
},
aggs: {
cardinality_count: {
cardinality: {
field: 'source.ip',
},
},
cardinality_check: {
bucket_selector: {
buckets_path: {
cardinalityCount: 'cardinality_count',
},
script: 'params.cardinalityCount >= 5',
},
},
top_threshold_hits: {
top_hits: {
sort: [
{
'@timestamp': {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
},
},
},
},
})
);
});
});

147
x-pack/plugins/security_solution/server/lib/detection_engine/signals/find_threshold_signals.ts
View file

@ -6,12 +6,12 @@
*/
import { set } from '@elastic/safer-lodash-set';
import { isEmpty } from 'lodash/fp';
import {
Threshold,
TimestampOverrideOrUndefined,
} from '../../../../common/detection_engine/schemas/common/schemas';
import { normalizeThresholdField } from '../../../../common/detection_engine/utils';
import { singleSearchAfter } from './single_search_after';
import {
@ -50,69 +50,98 @@ export const findThresholdSignals = async ({
searchDuration: string;
searchErrors: string[];
}> => {
const thresholdFields = Array.isArray(threshold.field) ? threshold.field : [threshold.field];
const topHitsAgg = {
top_hits: {
sort: [
{
[timestampOverride ?? '@timestamp']: {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
},
};
const aggregations =
threshold && !isEmpty(threshold.field)
? thresholdFields.reduce((acc, field, i) => {
const aggPath = [...Array(i + 1).keys()]
.map((j) => {
return `['threshold_${j}:${thresholdFields[j]}']`;
})
.join(`['aggs']`);
set(acc, aggPath, {
terms: {
field,
min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set
size: 10000, // max 10k buckets
},
});
if (i === threshold.field.length - 1) {
const topHitsAgg = {
top_hits: {
sort: [
{
[timestampOverride ?? '@timestamp']: {
order: 'desc',
},
},
],
fields: [
{
field: '*',
include_unmapped: true,
},
],
size: 1,
const thresholdFields = normalizeThresholdField(threshold.field);
const aggregations = thresholdFields.length
? thresholdFields.reduce((acc, field, i) => {
const aggPath = [...Array(i + 1).keys()]
.map((j) => {
return `['threshold_${j}:${thresholdFields[j]}']`;
})
.join(`['aggs']`);
set(acc, aggPath, {
terms: {
field,
min_doc_count: threshold.value, // not needed on parent agg, but can help narrow down result set
size: 10000, // max 10k buckets
},
});
if (i === (thresholdFields.length ?? 0) - 1) {
if (threshold.cardinality?.length) {
set(acc, `${aggPath}['aggs']`, {
top_threshold_hits: topHitsAgg,
cardinality_count: {
cardinality: {
field: threshold.cardinality[0].field,
},
},
};
// TODO: support case where threshold fields are not supplied, but cardinality is?
if (!isEmpty(threshold.cardinality_field)) {
set(acc, `${aggPath}['aggs']`, {
top_threshold_hits: topHitsAgg,
cardinality_count: {
cardinality: {
field: threshold.cardinality_field,
cardinality_check: {
bucket_selector: {
buckets_path: {
cardinalityCount: 'cardinality_count',
},
script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator
},
cardinality_check: {
bucket_selector: {
buckets_path: {
cardinalityCount: 'cardinality_count',
},
script: `params.cardinalityCount >= ${threshold.cardinality_value}`, // TODO: cardinality operator
},
},
});
} else {
set(acc, `${aggPath}['aggs']`, {
top_threshold_hits: topHitsAgg,
});
}
},
});
} else {
set(acc, `${aggPath}['aggs']`, {
top_threshold_hits: topHitsAgg,
});
}
return acc;
}, {})
: {};
}
return acc;
}, {})
: {
threshold_0: {
terms: {
script: {
source: '""',
lang: 'painless',
},
min_doc_count: threshold.value,
},
aggs: {
top_threshold_hits: topHitsAgg,
...(threshold.cardinality?.length
? {
cardinality_count: {
cardinality: {
field: threshold.cardinality[0].field,
},
},
cardinality_check: {
bucket_selector: {
buckets_path: {
cardinalityCount: 'cardinality_count',
},
script: `params.cardinalityCount >= ${threshold.cardinality[0].value}`, // TODO: cardinality operator
},
},
}
: {}),
},
},
};
return singleSearchAfter({
aggregations,

52
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.test.ts
View file

@ -103,4 +103,56 @@ describe('signal_params_schema', () => {
const { falsePositives, ...withoutFalsePositives } = getSignalParamsSchemaMock();
expect(schema.validate(withoutFalsePositives).falsePositives).toEqual([]);
});
test('threshold validates with `value` only', () => {
const schema = signalParamsSchema();
const threshold = {
value: 200,
};
const mock = {
...getSignalParamsSchemaMock(),
threshold,
};
expect(schema.validate(mock).threshold?.value).toEqual(200);
});
test('threshold does not validate without `value`', () => {
const schema = signalParamsSchema();
const threshold = {
field: 'agent.id',
cardinality: [
{
field: ['host.name'],
value: 5,
},
],
};
const mock = {
...getSignalParamsSchemaMock(),
threshold,
};
expect(() => schema.validate(mock)).toThrow();
});
test('threshold `cardinality` cannot currently be greater than length 1', () => {
const schema = signalParamsSchema();
const threshold = {
value: 100,
cardinality: [
{
field: 'host.name',
value: 5,
},
{
field: 'user.name',
value: 5,
},
],
};
const mock = {
...getSignalParamsSchemaMock(),
threshold,
};
expect(() => schema.validate(mock)).toThrow();
});
});

13
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_params_schema.ts
View file

@ -41,10 +41,19 @@ export const signalSchema = schema.object({
threat: schema.nullable(schema.arrayOf(schema.object({}, { unknowns: 'allow' }))),
threshold: schema.maybe(
schema.object({
// Can be an empty string or empty array
field: schema.nullable(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
// Always required
value: schema.number(),
cardinality_field: schema.nullable(schema.string()), // TODO: depends on `field` being defined?
cardinality_value: schema.nullable(schema.number()),
cardinality: schema.nullable(
schema.arrayOf(
schema.object({
field: schema.string(),
value: schema.number(),
}),
{ maxSize: 1 }
)
),
})
),
timestampOverride: schema.nullable(schema.string()),

7
x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts
View file

@ -25,6 +25,7 @@ import {
isEqlRule,
isThreatMatchRule,
hasLargeValueItem,
normalizeThresholdField,
} from '../../../../common/detection_engine/utils';
import { parseScheduleDates } from '../../../../common/detection_engine/parse_schedule_dates';
import { SetupPlugins } from '../../../plugin';
@ -374,10 +375,6 @@ export const signalRulesAlertType = ({
}
const inputIndex = await getInputIndex(services, version, index);
const thresholdFields = Array.isArray(threshold.field)
? threshold.field
: [threshold.field];
const {
filters: bucketFilters,
searchErrors: previousSearchErrors,
@ -388,7 +385,7 @@ export const signalRulesAlertType = ({
services,
logger,
ruleId,
bucketByFields: thresholdFields,
bucketByFields: normalizeThresholdField(threshold.field),
timestampOverride,
buildRuleMessage,
});

133
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.test.ts
View file

@ -6,7 +6,11 @@
*/
import { alertsMock, AlertServicesMock } from '../../../../../alerts/server/mocks';
import { mockLogger, sampleWrappedThresholdSignalHit } from './__mocks__/es_results';
import {
mockLogger,
sampleWrappedThresholdSignalHit,
sampleWrappedLegacyThresholdSignalHit,
} from './__mocks__/es_results';
import { getThresholdBucketFilters } from './threshold_get_bucket_filters';
import { buildRuleMessageFactory } from './rule_messages';
@ -82,4 +86,131 @@ describe('thresholdGetBucketFilters', () => {
searchErrors: [],
});
});
it('should generate filters for threshold signal detection based on pre-7.12 signals', async () => {
mockService.callCluster.mockResolvedValue({
took: 10,
timed_out: false,
_shards: {
total: 10,
successful: 10,
failed: 0,
skipped: 0,
},
hits: {
total: 1,
max_score: 100,
hits: [sampleWrappedLegacyThresholdSignalHit()],
},
});
const result = await getThresholdBucketFilters({
from: 'now-6m',
to: 'now',
indexPattern: ['*'],
services: mockService,
logger: mockLogger,
ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
bucketByFields: ['host.name'],
timestampOverride: undefined,
buildRuleMessage,
});
expect(result).toEqual({
filters: [
{
bool: {
must_not: [
{
bool: {
filter: [
{
range: {
'@timestamp': {
lte: '2021-02-16T17:37:34.275Z',
},
},
},
{
term: {
'host.name': 'a hostname',
},
},
],
},
},
],
},
},
],
searchErrors: [],
});
});
it('should generate filters for threshold signal detection with mixed pre-7.12 and post-7.12 signals', async () => {
const signalHit = sampleWrappedThresholdSignalHit();
const wrappedSignalHit = {
...signalHit,
_source: {
...signalHit._source,
signal: {
...signalHit._source.signal,
original_time: '2021-02-16T18:37:34.275Z',
},
},
};
mockService.callCluster.mockResolvedValue({
took: 10,
timed_out: false,
_shards: {
total: 10,
successful: 10,
failed: 0,
skipped: 0,
},
hits: {
total: 1,
max_score: 100,
hits: [sampleWrappedLegacyThresholdSignalHit(), wrappedSignalHit],
},
});
const result = await getThresholdBucketFilters({
from: 'now-6m',
to: 'now',
indexPattern: ['*'],
services: mockService,
logger: mockLogger,
ruleId: '7a7065d7-6e8b-4aae-8d20-c93613dec9f9',
bucketByFields: ['host.name'],
timestampOverride: undefined,
buildRuleMessage,
});
expect(result).toEqual({
filters: [
{
bool: {
must_not: [
{
bool: {
filter: [
{
range: {
'@timestamp': {
lte: '2021-02-16T18:37:34.275Z',
},
},
},
{
term: {
'host.name': 'a hostname',
},
},
],
},
},
],
},
},
],
searchErrors: [],
});
});
});

5
x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold_get_bucket_filters.ts
View file

@ -74,8 +74,9 @@ export const getThresholdBucketFilters = async ({
if (signalTerms == null) {
signalTerms = [
{
field: (((hit._source.rule as RulesSchema).threshold as unknown) as { field: string })
.field,
field: (((hit._source.signal?.rule as RulesSchema).threshold as unknown) as {
field: string;
}).field,
value: ((hit._source.signal?.threshold_result as unknown) as { value: string }).value,
},
];