maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[SIEM][Detection Engine] Removes filter type, fixes bugs, adds more examples (#52452)

Browse source 
## Summary

* This removes the filter type and all the tests associated with it.
* This fixes a critical bug where filter from params was being passed down instead of esFilter
* This adds and cleans up all the rule examples for documenters and developers and users.
* This fixes a bug with empty queries
* This makes it so you can have simple filters which replace the filter capability
* This cleans up info and debug messages more

### Checklist

Use ~~strikethroughs~~ to remove checklist items you don't feel are applicable to this PR.

~~- [ ] This was checked for cross-browser compatibility, [including a check against IE11](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility)~~

~~- [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md)~~

- [x] [Documentation](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#writing-documentation) was added for features that require explanation or tutorials
- [x] [Unit or functional tests](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#cross-browser-compatibility) were updated or added to match the most common scenarios

~~- [ ] This was checked for [keyboard-only and screenreader accessibility](https://developer.mozilla.org/en-US/docs/Learn/Tools_and_testing/Cross_browser_testing/Accessibility#Accessibility_testing_checklist)~~

### For maintainers

~~- [ ] This was checked for breaking API changes and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)~~

- [x] This includes a feature addition or change that requires a release note and was [labeled appropriately](https://github.com/elastic/kibana/blob/master/CONTRIBUTING.md#release-notes-process)
This commit is contained in:
Frank Hassanabad 2019-12-09 12:39:06 -07:00 committed by GitHub
parent 77dca06253
commit 4bbe3cf85b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
71 changed files with 642 additions and 783 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md
View file

@ -60,7 +60,7 @@ which will:
- Delete any existing alert tasks you have
- Delete any existing signal mapping, policies, and template, you might have previously had.
- Add the latest signal index and its mappings using your settings from `kibana.dev.yml` environment variable of `xpack.siem.signalsIndex`.
- Posts the sample rule from `rules/root_or_admin_1.json`
- Posts the sample rule from `./rules/queries/query_with_rule_id.json`
- The sample rule checks for root or admin every 5 minutes and reports that as a signal if it is a positive hit
Now you can run
@ -154,11 +154,12 @@ logging.events:
See these two README.md's pages for more references on the alerting and actions API:
https://github.com/elastic/kibana/blob/master/x-pack/legacy/plugins/alerting/README.md
https://github.com/elastic/kibana/tree/master/x-pack/legacy/plugins/actions
### Signals API
To update the status of a signal or group of signals, the following scripts provide an example of how to
go about doing so.
` cd x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts`
`cd x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts`
`./signals/put_signal_doc.sh` will post a sample signal doc into the signals index to play with
`./signals/set_status_with_id.sh closed` will update the status of the sample signal to closed
`./signals/set_status_with_id.sh open` will update the status of the sample signal to open

1
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts
View file

@ -31,7 +31,6 @@ export const sampleRuleAlertParams = (
references: ['http://google.com'],
riskScore: riskScore ? riskScore : 50,
maxSignals: maxSignals ? maxSignals : 10000,
filter: undefined,
filters: undefined,
savedId: undefined,
meta: undefined,

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_rules.ts
View file

@ -13,7 +13,6 @@ export const createRules = async ({
description,
enabled,
falsePositives,
filter,
from,
query,
language,
@ -46,7 +45,6 @@ export const createRules = async ({
index,
falsePositives,
from,
filter,
immutable,
query,
language,

23
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.test.ts
View file

@ -385,26 +385,9 @@ describe('get_filter', () => {
});
describe('getFilter', () => {
test('returns a filter if given a type of filter as is', async () => {
const filter = await getFilter({
type: 'filter',
filter: { something: '1' },
filters: undefined,
language: undefined,
query: undefined,
savedId: undefined,
services: servicesMock,
index: ['auditbeat-*'],
});
expect(filter).toEqual({
something: '1',
});
});
test('returns a query if given a type of query', async () => {
const filter = await getFilter({
type: 'query',
filter: undefined,
filters: undefined,
language: 'kuery',
query: 'host.name: siem',
@ -439,7 +422,6 @@ describe('get_filter', () => {
await expect(
getFilter({
type: 'query',
filter: undefined,
filters: undefined,
language: undefined,
query: 'host.name: siem',
@ -454,7 +436,6 @@ describe('get_filter', () => {
await expect(
getFilter({
type: 'query',
filter: undefined,
filters: undefined,
language: 'kuery',
query: undefined,
@ -469,7 +450,6 @@ describe('get_filter', () => {
await expect(
getFilter({
type: 'query',
filter: undefined,
filters: undefined,
language: 'kuery',
query: 'host.name: siem',
@ -483,7 +463,6 @@ describe('get_filter', () => {
test('returns a saved query if given a type of query', async () => {
const filter = await getFilter({
type: 'saved_query',
filter: undefined,
filters: undefined,
language: undefined,
query: undefined,
@ -507,7 +486,6 @@ describe('get_filter', () => {
await expect(
getFilter({
type: 'saved_query',
filter: undefined,
filters: undefined,
language: undefined,
query: undefined,
@ -522,7 +500,6 @@ describe('get_filter', () => {
await expect(
getFilter({
type: 'saved_query',
filter: undefined,
filters: undefined,
language: undefined,
query: undefined,

5
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_filter.ts
View file

@ -42,7 +42,6 @@ export const getQueryFilter = (
interface GetFilterArgs {
type: RuleAlertParams['type'];
filter: Record<string, {}> | undefined | null;
filters: PartialFilter[] | undefined | null;
language: string | undefined | null;
query: string | undefined | null;
@ -52,7 +51,6 @@ interface GetFilterArgs {
}
export const getFilter = async ({
filter,
filters,
index,
language,
@ -95,9 +93,6 @@ export const getFilter = async ({
throw new TypeError('savedId parameter should be defined');
}
}
case 'filter': {
return filter;
}
}
return assertUnreachable(type);
};

24
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/rules_alert_type.ts
View file

@ -34,7 +34,6 @@ export const rulesAlertType = ({
description: schema.string(),
falsePositives: schema.arrayOf(schema.string(), { defaultValue: [] }),
from: schema.string(),
filter: schema.nullable(schema.object({}, { allowUnknowns: true })),
ruleId: schema.string(),
immutable: schema.boolean({ defaultValue: false }),
index: schema.nullable(schema.arrayOf(schema.string())),
@ -56,7 +55,6 @@ export const rulesAlertType = ({
},
async executor({ alertId, services, params }) {
const {
filter,
from,
ruleId,
index,
@ -87,7 +85,6 @@ export const rulesAlertType = ({
const inputIndex = await getInputIndex(services, version, index);
const esFilter = await getFilter({
type,
filter,
filters,
language,
query,
@ -106,18 +103,20 @@ export const rulesAlertType = ({
});
try {
logger.debug(`Starting signal rule "id: ${alertId}", "ruleId: ${ruleId}"`);
logger.debug(
`[+] Initial search call of signal rule "id: ${alertId}", "ruleId: ${ruleId}"`
`Starting signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
logger.debug(
`[+] Initial search call of signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
const noReIndexResult = await services.callCluster('search', noReIndex);
if (noReIndexResult.hits.total.value !== 0) {
logger.info(
`Found ${
noReIndexResult.hits.total.value
} signals from the indexes of "${inputIndex.join(
} signals from the indexes of "[${inputIndex.join(
', '
)}" using signal rule "id: ${alertId}", "ruleId: ${ruleId}", pushing signals to index ${outputIndex}`
)}]" using signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}", pushing signals to index "${outputIndex}"`
);
}
@ -128,6 +127,7 @@ export const rulesAlertType = ({
logger,
id: alertId,
signalsIndex: outputIndex,
filter: esFilter,
name,
createdBy,
updatedBy,
@ -137,15 +137,19 @@ export const rulesAlertType = ({
});
if (bulkIndexResult) {
logger.debug(`Finished signal rule "id: ${alertId}", "ruleId: ${ruleId}"`);
logger.debug(
`Finished signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
} else {
logger.error(`Error processing signal rule "id: ${alertId}", "ruleId: ${ruleId}"`);
logger.error(
`Error processing signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
}
} catch (err) {
// TODO: Error handling and writing of errors into a signal that has error
// handling/conditions
logger.error(
`Error from signal rule "id: ${alertId}", "ruleId: ${ruleId}", ${err.message}`
`Error from signal rule name: "${name}", id: "${alertId}", rule_id: "${ruleId}"`
);
}
},

3
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts
View file

@ -35,7 +35,6 @@ export interface RuleAlertParams {
description: string;
enabled: boolean;
falsePositives: string[];
filter: Record<string, {}> | undefined | null;
filters: PartialFilter[] | undefined | null;
from: string;
immutable: boolean;
@ -55,7 +54,7 @@ export interface RuleAlertParams {
tags: string[];
to: string;
threats: ThreatParams[] | undefined | null;
type: 'filter' | 'query' | 'saved_query';
type: 'query' | 'saved_query';
}
export type RuleAlertParamsRest = Omit<

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/update_rules.ts
View file

@ -53,7 +53,6 @@ export const updateRules = async ({
savedId,
meta,
filters,
filter,
from,
immutable,
id,
@ -88,7 +87,6 @@ export const updateRules = async ({
{
description,
falsePositives,
filter,
from,
immutable,
query,

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts
View file

@ -530,6 +530,7 @@ describe('utils', () => {
services: mockService,
logger: mockLogger,
pageSize: 1,
filter: undefined,
})
).rejects.toThrow('Attempted to search after with empty sort id');
});
@ -543,6 +544,7 @@ describe('utils', () => {
services: mockService,
logger: mockLogger,
pageSize: 1,
filter: undefined,
});
expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId);
});
@ -559,6 +561,7 @@ describe('utils', () => {
services: mockService,
logger: mockLogger,
pageSize: 1,
filter: undefined,
})
).rejects.toThrow('Fake Error');
});
@ -579,6 +582,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(mockService.callCluster).toHaveBeenCalledTimes(0);
expect(result).toEqual(true);
@ -629,6 +633,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(mockService.callCluster).toHaveBeenCalledTimes(5);
expect(result).toEqual(true);
@ -650,6 +655,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(mockLogger.error).toHaveBeenCalled();
expect(result).toEqual(false);
@ -678,6 +684,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(mockLogger.error).toHaveBeenCalled();
expect(result).toEqual(false);
@ -706,6 +713,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(result).toEqual(true);
});
@ -736,6 +744,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(result).toEqual(true);
});
@ -766,6 +775,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(result).toEqual(true);
});
@ -798,6 +808,7 @@ describe('utils', () => {
interval: '5m',
enabled: true,
pageSize: 1,
filter: undefined,
});
expect(result).toEqual(false);
});

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts
View file

@ -47,7 +47,6 @@ export const buildRule = ({
risk_score: ruleParams.riskScore,
output_index: ruleParams.outputIndex,
description: ruleParams.description,
filter: ruleParams.filter,
from: ruleParams.from,
immutable: ruleParams.immutable,
index: ruleParams.index,
@ -207,7 +206,9 @@ export const singleBulkCreate = async ({
body: bulkBody,
});
const time2 = performance.now();
logger.debug(`individual bulk process time took: ${time2 - time1} milliseconds`);
logger.debug(
`individual bulk process time took: ${Number(time2 - time1).toFixed(2)} milliseconds`
);
logger.debug(`took property says bulk took: ${firstResult.took} milliseconds`);
if (firstResult.errors) {
// go through the response status errors and see what
@ -241,6 +242,7 @@ interface SingleSearchAfterParams {
services: AlertServices;
logger: Logger;
pageSize: number;
filter: unknown;
}
// utilize search_after for paging results into bulk.
@ -248,6 +250,7 @@ export const singleSearchAfter = async ({
searchAfterSortId,
ruleParams,
services,
filter,
logger,
pageSize,
}: SingleSearchAfterParams): Promise<SignalSearchResponse> => {
@ -259,7 +262,7 @@ export const singleSearchAfter = async ({
index: ruleParams.index,
from: ruleParams.from,
to: ruleParams.to,
filter: ruleParams.filter,
filter,
size: pageSize,
searchAfterSortId,
});
@ -287,6 +290,7 @@ interface SearchAfterAndBulkCreateParams {
interval: string;
enabled: boolean;
pageSize: number;
filter: unknown;
}
// search_after through documents and re-index using bulk endpoint.
@ -297,6 +301,7 @@ export const searchAfterAndBulkCreate = async ({
logger,
id,
signalsIndex,
filter,
name,
createdBy,
updatedBy,
@ -353,6 +358,7 @@ export const searchAfterAndBulkCreate = async ({
ruleParams,
services,
logger,
filter,
pageSize, // maximum number of docs to receive per search result.
});
if (searchAfterResult.hits.hits.length === 0) {

14
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/__mocks__/request_responses.ts
View file

@ -38,20 +38,6 @@ export const typicalPayload = (): Partial<Omit<RuleAlertParamsRest, 'filter'>> =
],
});
export const typicalFilterPayload = (): Partial<RuleAlertParamsRest> => ({
rule_id: 'rule-1',
description: 'Detecting root and admin users',
index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'],
interval: '5m',
name: 'Detect Root/Admin Users',
risk_score: 50,
type: 'filter',
from: 'now-6m',
to: 'now',
severity: 'high',
filter: {},
});
export const typicalSetStatusSignalByIdsPayload = (): Partial<SignalsRestParams> => ({
signal_ids: ['somefakeid1', 'somefakeid2'],
status: 'closed',

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.test.ts
View file

@ -103,27 +103,6 @@ describe('create_rules', () => {
expect(statusCode).toBe(200);
});
test('returns 200 if type is filter', async () => {
alertsClient.find.mockResolvedValue(getFindResult());
alertsClient.get.mockResolvedValue(getResult());
actionsClient.create.mockResolvedValue(createActionResult());
alertsClient.create.mockResolvedValue(getResult());
// Cannot type request with a ServerInjectOptions as the type system complains
// about the property filter involving Hapi types, so I left it off for now
const { language, query, type, ...noType } = typicalPayload();
const request = {
method: 'POST',
url: DETECTION_ENGINE_RULES_URL,
payload: {
...noType,
type: 'filter',
filter: {},
},
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(200);
});
test('returns 400 if type is not filter or kql', async () => {
alertsClient.find.mockResolvedValue(getFindResult());
alertsClient.get.mockResolvedValue(getResult());

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_rules_route.ts
View file

@ -35,7 +35,6 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute =
description,
enabled,
false_positives: falsePositives,
filter,
from,
immutable,
query,
@ -81,7 +80,7 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute =
if (ruleId != null) {
const rule = await readRules({ alertsClient, ruleId });
if (rule != null) {
return new Boom(`rule_id ${ruleId} already exists`, { statusCode: 409 });
return new Boom(`rule_id: "${ruleId}" already exists`, { statusCode: 409 });
}
}
const createdRule = await createRules({
@ -90,7 +89,6 @@ export const createCreateRulesRoute = (server: ServerFacade): Hapi.ServerRoute =
description,
enabled,
falsePositives,
filter,
from,
immutable,
query,

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/create_index_route.ts
View file

@ -37,7 +37,7 @@ export const createCreateIndexRoute = (server: ServerFacade): Hapi.ServerRoute =
const callWithRequest = callWithRequestFactory(request);
const indexExists = await getIndexExists(callWithRequest, index);
if (indexExists) {
return new Boom(`index ${index} already exists`, { statusCode: 409 });
return new Boom(`index: "${index}" already exists`, { statusCode: 409 });
} else {
const policyExists = await getPolicyExists(callWithRequest, index);
if (!policyExists) {

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/delete_index_route.ts
View file

@ -45,7 +45,7 @@ export const createDeleteIndexRoute = (server: ServerFacade): Hapi.ServerRoute =
const callWithRequest = callWithRequestFactory(request);
const indexExists = await getIndexExists(callWithRequest, index);
if (!indexExists) {
return new Boom(`index ${index} does not exist`, { statusCode: 404 });
return new Boom(`index: "${index}" does not exist`, { statusCode: 404 });
} else {
await deleteAllIndex(callWithRequest, `${index}-*`);
const policyExists = await getPolicyExists(callWithRequest, index);

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/index/read_index_route.ts
View file

@ -42,7 +42,7 @@ export const createReadIndexRoute = (server: ServerFacade): Hapi.ServerRoute =>
if (request.method.toLowerCase() === 'head') {
return headers.response().code(404);
} else {
return new Boom('An index for this space does not exist', { statusCode: 404 });
return new Boom('index for this space does not exist', { statusCode: 404 });
}
}
} catch (err) {

290
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts
View file

@ -141,7 +141,7 @@ describe('schemas', () => {
).toBeTruthy();
});
test('[rule_id, description, from, to, name, severity, type, query, index, interval] does not validate', () => {
test('[rule_id, description, from, to, name, severity, type, query, index, interval] does validate', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
@ -156,7 +156,7 @@ describe('schemas', () => {
index: ['index-1'],
interval: '5m',
}).error
).toBeTruthy();
).toBeFalsy();
});
test('[rule_id, description, from, to, index, name, severity, interval, type, query, language] does not validate', () => {
@ -227,8 +227,7 @@ describe('schemas', () => {
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
type: 'query',
risk_score: 50,
}).error
).toBeFalsy();
@ -247,8 +246,7 @@ describe('schemas', () => {
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
type: 'query',
}).error
).toBeFalsy();
});
@ -287,8 +285,7 @@ describe('schemas', () => {
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
type: 'query',
threats: [
{
framework: 'someFramework',
@ -310,84 +307,6 @@ describe('schemas', () => {
).toBeFalsy();
});
test('If filter type is set then filter is required', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
}).error
).toBeTruthy();
});
test('If filter type is set then query is not allowed', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
query: 'some query value',
}).error
).toBeTruthy();
});
test('If filter type is set then language is not allowed', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
language: 'kuery',
}).error
).toBeTruthy();
});
test('If filter type is set then filters are not allowed', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
filters: [],
}).error
).toBeTruthy();
});
test('allows references to be sent as valid', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
@ -509,26 +428,6 @@ describe('schemas', () => {
).toEqual(100);
});
test('filter and filters cannot exist together', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'query',
filter: {},
filters: [],
}).error
).toBeTruthy();
});
test('saved_id is required when type is saved_query and will not validate without out', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
@ -608,26 +507,6 @@ describe('schemas', () => {
).toBeTruthy();
});
test('saved_query type cannot have filter with it', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
rule_id: 'rule-1',
risk_score: 50,
output_index: '.siem-signals',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'saved_query',
saved_id: 'some id',
filter: {},
}).error
).toBeTruthy();
});
test('language validates with kuery', () => {
expect(
createRulesSchema.validate<Partial<RuleAlertParamsRest>>({
@ -1157,30 +1036,6 @@ describe('schemas', () => {
).toBeTruthy();
});
test('You can have an empty query string when filters are present', () => {
expect(
createRulesSchema.validate<Partial<Omit<RuleAlertParamsRest, 'meta'> & { meta: string }>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
immutable: true,
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'query',
references: ['index-1'],
query: '',
language: 'kuery',
filters: [],
max_signals: 1,
}).error
).toBeFalsy();
});
test('You can omit the query string when filters are present', () => {
expect(
createRulesSchema.validate<Partial<Omit<RuleAlertParamsRest, 'meta'> & { meta: string }>>({
@ -1203,29 +1058,6 @@ describe('schemas', () => {
}).error
).toBeFalsy();
});
test('query string defaults to empty string when present with filters', () => {
expect(
createRulesSchema.validate<Partial<Omit<RuleAlertParamsRest, 'meta'> & { meta: string }>>({
rule_id: 'rule-1',
output_index: '.siem-signals',
risk_score: 50,
description: 'some description',
from: 'now-5m',
to: 'now',
immutable: true,
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'query',
references: ['index-1'],
language: 'kuery',
filters: [],
max_signals: 1,
}).value.query
).toEqual('');
});
});
describe('update rules schema', () => {
@ -1556,8 +1388,7 @@ describe('schemas', () => {
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
type: 'query',
}).error
).toBeFalsy();
});
@ -1573,82 +1404,11 @@ describe('schemas', () => {
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
type: 'query',
}).error
).toBeFalsy();
});
test('If filter type is set then filter is still not required', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
}).error
).toBeFalsy();
});
test('If filter type is set then query is not allowed', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
query: 'some query value',
}).error
).toBeTruthy();
});
test('If filter type is set then language is not allowed', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
language: 'kuery',
}).error
).toBeTruthy();
});
test('If filter type is set then filters are not allowed', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'filter',
filter: {},
filters: [],
}).error
).toBeTruthy();
});
test('allows references to be sent as a valid value to update with', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
@ -1758,24 +1518,6 @@ describe('schemas', () => {
).toBeTruthy();
});
test('filter and filters cannot exist together', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'query',
filter: {},
filters: [],
}).error
).toBeTruthy();
});
test('saved_id is not required when type is saved_query and will validate without it', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
@ -1827,24 +1569,6 @@ describe('schemas', () => {
).toBeFalsy();
});
test('saved_query type cannot have filter with it', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({
id: 'rule-1',
description: 'some description',
from: 'now-5m',
to: 'now',
index: ['index-1'],
name: 'some-name',
severity: 'severity',
interval: '5m',
type: 'saved_query',
saved_id: 'some id',
filter: {},
}).error
).toBeTruthy();
});
test('language validates with kuery', () => {
expect(
updateRulesSchema.validate<Partial<UpdateRuleAlertParamsRest>>({

78
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts
View file

@ -11,7 +11,6 @@ import { DEFAULT_MAX_SIGNALS } from '../../../../common/constants';
const description = Joi.string();
const enabled = Joi.boolean();
const false_positives = Joi.array().items(Joi.string());
const filter = Joi.object();
const filters = Joi.array();
const from = Joi.string();
const immutable = Joi.boolean();
@ -34,7 +33,7 @@ const risk_score = Joi.number()
const severity = Joi.string();
const status = Joi.string().valid('open', 'closed');
const to = Joi.string();
const type = Joi.string().valid('filter', 'query', 'saved_query');
const type = Joi.string().valid('query', 'saved_query');
const queryFilter = Joi.string();
const references = Joi.array()
.items(Joi.string())
@ -85,46 +84,14 @@ export const createRulesSchema = Joi.object({
description: description.required(),
enabled: enabled.default(true),
false_positives: false_positives.default([]),
filter: filter.when('type', { is: 'filter', then: Joi.required(), otherwise: Joi.forbidden() }),
filters: Joi.when('type', {
is: 'query',
then: filters.optional(),
otherwise: Joi.when('type', {
is: 'saved_query',
then: filters.optional(),
otherwise: Joi.forbidden(),
}),
}),
filters,
from: from.required(),
rule_id,
immutable: immutable.default(false),
index,
interval: interval.default('5m'),
query: Joi.when('type', {
is: 'query',
then: Joi.when('filters', {
is: Joi.exist(),
then: query
.optional()
.allow('')
.default(''),
otherwise: Joi.required(),
}),
otherwise: Joi.when('type', {
is: 'saved_query',
then: query.optional(),
otherwise: Joi.forbidden(),
}),
}),
language: Joi.when('type', {
is: 'query',
then: language.required(),
otherwise: Joi.when('type', {
is: 'saved_query',
then: language.optional(),
otherwise: Joi.forbidden(),
}),
}),
query: query.allow('').default(''),
language: language.default('kuery'),
output_index,
saved_id: saved_id.when('type', {
is: 'saved_query',
@ -147,46 +114,17 @@ export const updateRulesSchema = Joi.object({
description,
enabled,
false_positives,
filter: filter.when('type', { is: 'filter', then: Joi.optional(), otherwise: Joi.forbidden() }),
filters: Joi.when('type', {
is: 'query',
then: filters.optional(),
otherwise: Joi.when('type', {
is: 'saved_query',
then: filters.optional(),
otherwise: Joi.forbidden(),
}),
}),
filters,
from,
rule_id,
id,
immutable,
index,
interval,
query: Joi.when('type', {
is: 'query',
then: query.optional(),
otherwise: Joi.when('type', {
is: 'saved_query',
then: query.optional(),
otherwise: Joi.forbidden(),
}),
}),
language: Joi.when('type', {
is: 'query',
then: language.optional(),
otherwise: Joi.when('type', {
is: 'saved_query',
then: language.optional(),
otherwise: Joi.forbidden(),
}),
}),
query: query.allow(''),
language,
output_index,
saved_id: saved_id.when('type', {
is: 'saved_query',
then: Joi.optional(),
otherwise: Joi.forbidden(),
}),
saved_id,
meta,
risk_score,
max_signals,

15
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_rules_route.test.ts
View file

@ -20,7 +20,6 @@ import {
getUpdateRequest,
typicalPayload,
getFindResultWithSingleHit,
typicalFilterPayload,
} from './__mocks__/request_responses';
import { DETECTION_ENGINE_RULES_URL } from '../../../../common/constants';
@ -119,20 +118,6 @@ describe('update_rules', () => {
expect(statusCode).toBe(200);
});
test('returns 200 if type is filter', async () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());
actionsClient.update.mockResolvedValue(updateActionResult());
alertsClient.update.mockResolvedValue(getResult());
const request: ServerInjectOptions = {
method: 'PUT',
url: DETECTION_ENGINE_RULES_URL,
payload: typicalFilterPayload(),
};
const { statusCode } = await server.inject(request);
expect(statusCode).toBe(200);
});
test('returns 400 if type is not filter or kql', async () => {
alertsClient.find.mockResolvedValue(getFindResultWithSingleHit());
alertsClient.get.mockResolvedValue(getResult());

2
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_rules_route.ts
View file

@ -30,7 +30,6 @@ export const createUpdateRulesRoute: Hapi.ServerRoute = {
description,
enabled,
false_positives: falsePositives,
filter,
from,
immutable,
query,
@ -68,7 +67,6 @@ export const createUpdateRulesRoute: Hapi.ServerRoute = {
description,
enabled,
falsePositives,
filter,
from,
immutable,
query,

8
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts
View file

@ -341,22 +341,22 @@ describe('utils', () => {
describe('getIdError', () => {
test('outputs message about id not being found if only id is defined and ruleId is undefined', () => {
const boom = getIdError({ id: '123', ruleId: undefined });
expect(boom.message).toEqual('id of 123 not found');
expect(boom.message).toEqual('id: "123" not found');
});
test('outputs message about id not being found if only id is defined and ruleId is null', () => {
const boom = getIdError({ id: '123', ruleId: null });
expect(boom.message).toEqual('id of 123 not found');
expect(boom.message).toEqual('id: "123" not found');
});
test('outputs message about ruleId not being found if only ruleId is defined and id is undefined', () => {
const boom = getIdError({ id: undefined, ruleId: 'rule-id-123' });
expect(boom.message).toEqual('rule_id of rule-id-123 not found');
expect(boom.message).toEqual('rule_id: "rule-id-123" not found');
});
test('outputs message about ruleId not being found if only ruleId is defined and id is null', () => {
const boom = getIdError({ id: null, ruleId: 'rule-id-123' });
expect(boom.message).toEqual('rule_id of rule-id-123 not found');
expect(boom.message).toEqual('rule_id: "rule-id-123" not found');
});
test('outputs message about both being not defined when both are undefined', () => {

5
x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts
View file

@ -18,9 +18,9 @@ export const getIdError = ({
ruleId: string | undefined | null;
}) => {
if (id != null) {
return new Boom(`id of ${id} not found`, { statusCode: 404 });
return new Boom(`id: "${id}" not found`, { statusCode: 404 });
} else if (ruleId != null) {
return new Boom(`rule_id of ${ruleId} not found`, { statusCode: 404 });
return new Boom(`rule_id: "${ruleId}" not found`, { statusCode: 404 });
} else {
return new Boom(`id or rule_id should have been defined`, { statusCode: 404 });
}
@ -34,7 +34,6 @@ export const transformAlertToRule = (alert: RuleAlertType): Partial<OutputRuleAl
description: alert.params.description,
enabled: alert.enabled,
false_positives: alert.params.falsePositives,
filter: alert.params.filter,
filters: alert.params.filters,
from: alert.params.from,
id: alert.id,

6
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/post_rule.sh
View file

@ -10,11 +10,11 @@ set -e
./check_env_variables.sh
# Uses a default if no argument is specified
RULES=(${@:-./rules/root_or_admin_1.json})
RULES=(${@:-./rules/queries/query_with_rule_id.json})
# Example: ./post_rule.sh
# Example: ./post_rule.sh ./rules/root_or_admin_1.json
# Example glob: ./post_rule.sh ./rules/*
# Example: ./post_rule.sh ./rules/queries/query_with_rule_id.json
# Example glob: ./post_rule.sh ./rules/queries/*
for RULE in "${RULES[@]}"
do {
[ -e "$RULE" ] || continue

21
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/README.md Normal file
View file

@ -0,0 +1,21 @@
These are example POST rules that only have queries and and no filters in them.
Every single json file should have the field:
```sh
"type": "query"
```
set which is what designates it as a type of query.
To post all of them to see in the UI, with the scripts folder as your current working directory.
```sh
./post_rule.sh ./rules/queries/*.json
```
To post only one at a time:
```sh
./post_rule.sh ./rules/queries/<filename>.json
```

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_disabled.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Query which is disabled",
"description": "Example query which will is disabled and will not run after being posted",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"enabled": false
}

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_immutable.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Query which is immutable",
"description": "Example query which is immutable",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"immutable": true
}

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_lucene.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Query with the language set to lucene",
"description": "Query with the language set to lucene",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "lucene"
}

13
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_threats.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_mitre_attack.json
View file

@ -1,17 +1,12 @@
{
"rule_id": "rule-1",
"description": "Detecting root and admin users",
"index": ["auditbeat-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
"interval": "5s",
"name": "Detect Root/Admin Users",
"severity": "high",
"name": "Query which has Mitre Attack Data",
"description": "Example query which has Mitre Attack Data as threats",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6s",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"references": ["http://www.example.com", "https://ww.example.com"],
"threats": [
{
"framework": "MITRE ATT&CK",

80
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_with_everything.json Normal file
View file

@ -0,0 +1,80 @@
{
"name": "Query with all possible fields filled out",
"description": "Kitchen Sink (everything) query that has all possible fields filled out",
"false_positives": [
"https://www.example.com/some-article-about-a-false-positive",
"some text string about why another condition could be a false positive"
],
"rule_id": "rule-id-everything",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
},
{
"exists": {
"field": "host.hostname"
}
}
],
"enabled": false,
"immutable": true,
"index": ["auditbeat-*", "filebeat-*"],
"interval": "5m",
"query": "user.name: root or user.name: admin",
"output_index": ".siem-signals-default",
"meta": {
"anything_you_want_ui_related_or_otherwise": {
"as_deep_structured_as_you_need": {
"any_data_type": {}
}
}
},
"language": "kuery",
"risk_score": 1,
"max_signals": 100,
"tags": ["tag 1", "tag 2", "any tag you want"],
"to": "now",
"from": "now-6m",
"severity": "high",
"type": "query",
"threats": [
{
"framework": "MITRE ATT&CK",
"tactic": {
"id": "TA0040",
"name": "impact",
"reference": "https://attack.mitre.org/tactics/TA0040/"
},
"techniques": [
{
"id": "T1499",
"name": "endpoint denial of service",
"reference": "https://attack.mitre.org/techniques/T1499/"
}
]
},
{
"framework": "Some other Framework you want",
"tactic": {
"id": "some-other-id",
"name": "Some other name",
"reference": "https://example.com"
},
"techniques": [
{
"id": "some-other-id",
"name": "some other technique name",
"reference": "https://example.com"
}
]
}
],
"references": [
"http://www.example.com/some-article-about-attack",
"Some plain text string here explaining why this is a valid thing to look out for"
]
}

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_5.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_with_filter.json
View file

@ -1,15 +1,13 @@
{
"rule_id": "rule-5",
"risk_score": 5,
"description": "Detecting root and admin users over 24 hours on windows",
"interval": "5m",
"name": "Detect Root/Admin Users",
"name": "Query with two filters",
"description": "A KQL Query with a two filters",
"rule_id": "query-with-two-filters",
"risk_score": 15,
"severity": "high",
"type": "query",
"from": "now-24h",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"filters": [
{
"query": {

17
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_with_meta_data.json Normal file
View file

@ -0,0 +1,17 @@
{
"name": "Query which has extra meta data",
"description": "Query which has extra meta data",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"meta": {
"whatever-you-want": {
"store-stateful-stuff-for-ui": {
"or-anything-else": true
}
}
}
}

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/query_with_rule_id.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Query with a rule id",
"description": "Query with a rule_id that acts like an external id",
"rule_id": "query-rule-id",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin"
}

18
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/simplest_filters.json Normal file
View file

@ -0,0 +1,18 @@
{
"name": "Simplest Filter",
"description": "Simplest filter with the least amount of fields required",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
}
]
}

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/queries/simplest_query.json Normal file
View file

@ -0,0 +1,10 @@
{
"name": "Simplest Query",
"description": "Simplest query with the least amount of fields required",
"risk_score": 1,
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin"
}

14
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_1.json
View file

@ -1,14 +0,0 @@
{
"rule_id": "rule-1",
"risk_score": 1,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"references": ["http://www.example.com", "https://ww.example.com"]
}

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_10.json
View file

@ -1,12 +0,0 @@
{
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"references": ["http://www.example.com", "https://ww.example.com"]
}

13
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_2.json
View file

@ -1,13 +0,0 @@
{
"rule_id": "rule-2",
"risk_score": 2,
"description": "Detecting root and admin users over a long period of time",
"interval": "24h",
"name": "Detect Root/Admin Users over a long period of time",
"severity": "high",
"type": "query",
"from": "now-1y",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery"
}

13
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_3.json
View file

@ -1,13 +0,0 @@
{
"rule_id": "rule-3",
"risk_score": 3,
"description": "Detecting root and admin users as an empty set",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-16y",
"to": "now-15y",
"query": "user.name: root or user.name: admin",
"language": "kuery"
}

14
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_4.json
View file

@ -1,14 +0,0 @@
{
"rule_id": "rule-4",
"risk_score": 4,
"description": "Detecting root and admin users with lucene",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "lucene",
"references": ["http://www.example.com", "https://ww.example.com"]
}

15
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_8.json
View file

@ -1,15 +0,0 @@
{
"rule_id": "rule-8",
"risk_score": 8,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"enabled": false,
"references": ["http://www.example.com", "https://ww.example.com"]
}

18
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_9.json
View file

@ -1,18 +0,0 @@
{
"rule_id": "rule-9",
"risk_score": 9,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"enabled": false,
"tags": ["tag_1", "tag_2"],
"false_positives": ["false_1", "false_2"],
"immutable": true,
"references": ["http://www.example.com", "https://ww.example.com"]
}

46
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_filter_9998.json
View file

@ -1,46 +0,0 @@
{
"rule_id": "rule-9999",
"risk_score": 100,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "filter",
"from": "now-6m",
"to": "now",
"filter": {
"bool": {
"must": [],
"filter": [
{
"bool": {
"should": [
{
"match_phrase": {
"host.name": "siem-windows"
}
}
],
"minimum_should_match": 1
}
},
{
"match_phrase": {
"winlog.event_id": {
"query": "100"
}
}
},
{
"match_phrase": {
"agent.hostname": {
"query": "siem-windows"
}
}
}
],
"should": [],
"must_not": []
}
}
}

42
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_filter_9999.json
View file

@ -1,42 +0,0 @@
{
"rule_id": "rule-9999",
"risk_score": 100,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "filter",
"from": "now-6m",
"to": "now",
"filter": {
"bool": {
"should": [
{
"bool": {
"should": [
{
"match_phrase": {
"user.name": "root"
}
}
],
"minimum_should_match": 1
}
},
{
"bool": {
"should": [
{
"match_phrase": {
"user.name": "admin"
}
}
],
"minimum_should_match": 1
}
}
],
"minimum_should_match": 1
}
}
}

19
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_meta.json
View file

@ -1,19 +0,0 @@
{
"rule_id": "rule-meta-data",
"risk_score": 1,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery",
"references": ["http://www.example.com", "https://ww.example.com"],
"meta": {
"anything_i_want": {
"total_meta_for_ui_needs": true
}
}
}

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_saved_query_1.json
View file

@ -1,12 +0,0 @@
{
"rule_id": "saved-query-1",
"risk_score": 5,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"saved_id": "test-saveid"
}

14
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_saved_query_2.json
View file

@ -1,14 +0,0 @@
{
"rule_id": "saved-query-2",
"risk_score": 5,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"saved_id": "test-saveid-2",
"query": "user.name: root or user.name: admin",
"language": "kuery"
}

12
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_saved_query_3.json
View file

@ -1,12 +0,0 @@
{
"rule_id": "saved-query-3",
"risk_score": 5,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"saved_id": "test-saveid-3"
}

14
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_update_1.json
View file

@ -1,14 +0,0 @@
{
"rule_id": "rule-1",
"risk_score": 98,
"description": "Changed Description of only detecting root user",
"interval": "50m",
"name": "A different name",
"severity": "high",
"type": "query",
"from": "now-6m",
"to": "now-5m",
"query": "user.name: root",
"language": "kuery",
"references": ["https://update1.example.com", "https://update2.example.com"]
}

17
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_update_2.json
View file

@ -1,17 +0,0 @@
{
"rule_id": "rule-1",
"risk_score": 78,
"description": "Changed Description of only detecting root user",
"interval": "50m",
"name": "A different name",
"severity": "high",
"type": "query",
"false_positives": ["false_update_1", "false_update_2"],
"from": "now-6m",
"immutable": true,
"tags": ["some other tag for you"],
"to": "now-5m",
"query": "user.name: root",
"language": "kuery",
"references": ["https://update1.example.com", "https://update2.example.com"]
}

32
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/README.md Normal file
View file

@ -0,0 +1,32 @@
These are example POST rules that have the ability to use saved query id's and optionally
queries and filters. If you only use a saved query id, then detection engine relies on that
saved query id existing or it will throw errors if the user deletes the saved query id. If you
add a saved query id along side with a filter and/or query then it will try to use the saved query
id first and if that fails it will fall back on the provided filter and/or query.
Every single json file should have the field:
```sh
"type": "saved_query"
```
set which is what designates it as a type of saved_query
To post all of them to see in the UI, with the scripts folder as your current working directory:
```sh
./post_rule.sh ./rules/saved_queries/*.json
```
To post only one at a time:
```sh
./post_rule.sh ./rules/saved_queries/<filename>.json
```
If the saved_id does not exist and you do not provide a query and/or filter then expect to see this
in your kibana console logging:
```sh
server log [11:48:33.331] [error][task_manager] Task alerting:siem.signals "fedc2390-1858-11ea-9184-15f04d7099dc" failed: Error: Saved object [query/test-saved-id] not found
```

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/saved_query_by_rule_id.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Simplest Saved Query that uses a rule-id",
"rule_id": "simplest-saved-query",
"description": "Using a saved_id called test-saved-id but does not provide a pre-defined filter or query",
"risk_score": 5,
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"saved_id": "test-saved-id"
}

81
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/saved_query_with_everything.json Normal file
View file

@ -0,0 +1,81 @@
{
"name": "Saved Query with all possible fields filled out",
"description": "Kitchen Sink (everything) saved query that has all possible fields filled out",
"false_positives": [
"https://www.example.com/some-article-about-a-false-positive",
"some text string about why another condition could be a false positive"
],
"rule_id": "saved-query-everything",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
},
{
"exists": {
"field": "host.hostname"
}
}
],
"enabled": false,
"immutable": true,
"index": ["auditbeat-*", "filebeat-*"],
"interval": "5m",
"query": "user.name: root or user.name: admin",
"output_index": ".siem-signals-default",
"meta": {
"anything_you_want_ui_related_or_otherwise": {
"as_deep_structured_as_you_need": {
"any_data_type": {}
}
}
},
"language": "kuery",
"risk_score": 1,
"max_signals": 100,
"tags": ["tag 1", "tag 2", "any tag you want"],
"to": "now",
"from": "now-6m",
"severity": "high",
"type": "saved_query",
"threats": [
{
"framework": "MITRE ATT&CK",
"tactic": {
"id": "TA0040",
"name": "impact",
"reference": "https://attack.mitre.org/tactics/TA0040/"
},
"techniques": [
{
"id": "T1499",
"name": "endpoint denial of service",
"reference": "https://attack.mitre.org/techniques/T1499/"
}
]
},
{
"framework": "Some other Framework you want",
"tactic": {
"id": "some-other-id",
"name": "Some other name",
"reference": "https://example.com"
},
"techniques": [
{
"id": "some-other-id",
"name": "some other technique name",
"reference": "https://example.com"
}
]
}
],
"references": [
"http://www.example.com/some-article-about-attack",
"Some plain text string here explaining why this is a valid thing to look out for"
],
"saved_id": "test-saved-id"
}

19
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/saved_query_with_filters.json Normal file
View file

@ -0,0 +1,19 @@
{
"name": "Simplest saved id with filters",
"description": "Simplest saved_id with a filter for a fallback if the saved_id is not found",
"risk_score": 1,
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
}
],
"saved_id": "test-saved-id"
}

11
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/saved_query_with_query.json Normal file
View file

@ -0,0 +1,11 @@
{
"name": "Simplest Saved Query",
"description": "Simplest saved query with a fallback of a query",
"risk_score": 1,
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"query": "user.name: root or user.name: admin",
"saved_id": "test-saved-id"
}

26
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/saved_query_with_query_filter.json Normal file
View file

@ -0,0 +1,26 @@
{
"name": "Query with filter",
"description": "A KQL Query with a two filters",
"rule_id": "query-with-two-filters",
"risk_score": 15,
"severity": "high",
"type": "query",
"from": "now-24h",
"to": "now",
"query": "user.name: root or user.name: admin",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
},
{
"exists": {
"field": "host.hostname"
}
}
],
"saved_id": "test-saved-id"
}

10
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/saved_queries/simplest_saved_query.json Normal file
View file

@ -0,0 +1,10 @@
{
"name": "Simplest Saved Query",
"description": "Using a saved_id called test-saved-id but does not provide a pre-defined filter or query",
"risk_score": 5,
"severity": "high",
"type": "saved_query",
"from": "now-6m",
"to": "now",
"saved_id": "test-saved-id"
}

18
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/README.md Normal file
View file

@ -0,0 +1,18 @@
These are example POST rules that are more either Kibana UI test cases, mis-use test cases,
any generic e2e based test cases for testing different scenarios. Normally you would not
use these type of rule based messages when writing pure REST API calls. These messages are
more of what you would see "behind the scenes" when you are using Kibana UI which can
create rules with additional "meta" data or other implementation details that aren't really
a concern for a regular REST API user.
To post all of them to see in the UI, with the scripts folder as your current working directory:
```sh
./post_rule.sh ./rules/test_cases/*.json
```
To post only one at a time:
```sh
./post_rule.sh ./rules/test_cases/<filename>.json
```

0
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_with_empty_query.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/filter_with_empty_query.json
View file

0
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/filter_without_query.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/filter_without_query.json
View file

6
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_7.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/query_filter_ui_meatadata_lucene.json
View file

@ -1,9 +1,9 @@
{
"rule_id": "rule-7",
"name": "Query Filter With UI Meta Data With Lucene as a language",
"description": "A Query Filter that has lots of UI meta data state in the filters which goes back by 24 hours",
"rule_id": "query-filter-ui-lucene",
"risk_score": 7,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-24h",

6
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/root_or_admin_6.json → x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/query_filter_ui_metadata.json
View file

@ -1,9 +1,9 @@
{
"rule_id": "rule-6",
"name": "Query Filter With UI Meta Data",
"description": "A Query Filter that has lots of UI meta data state in the filters which goes back by 24 hours",
"rule_id": "query-filter-ui-kuery",
"risk_score": 6,
"description": "Detecting root and admin users",
"interval": "5m",
"name": "Detect Root/Admin Users",
"severity": "high",
"type": "query",
"from": "now-24h",

36
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/test_cases/saved_query_ui_meta_empty_query.json Normal file
View file

@ -0,0 +1,36 @@
{
"type": "saved_query",
"index": ["auditbeat-*", "endgame-*", "filebeat-*", "packetbeat-*", "winlogbeat-*"],
"language": "kuery",
"filters": [
{
"meta": {
"alias": null,
"negate": false,
"disabled": false,
"type": "phrase",
"key": "source.ip",
"params": { "query": "127.0.0.1" }
},
"query": {
"match": { "source.ip": { "query": "127.0.0.1", "type": "phrase" } }
},
"$state": { "store": "appState" }
}
],
"output_index": ".siem-signals-default",
"query": "",
"saved_id": "User's IP",
"false_positives": [],
"references": [],
"risk_score": 21,
"name": "Signal Maker 5000",
"description": "It's the Garrett",
"severity": "low",
"tags": ["Spong"],
"interval": "5m",
"from": "now-300s",
"enabled": true,
"to": "now",
"meta": { "from": "now-300s" }
}

25
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/README.md Normal file
View file

@ -0,0 +1,25 @@
These are example PUT rules to see how to update various parts of the rules.
You either have to use the id, or you have to use the rule_id in order to update
the rules. rule_id acts as an external_id where you can update rules across different
Kibana systems where id acts as a normal server generated id which is not normally shared
across different Kibana systems.
The only thing you cannot update is the `rule_id` or regular `id` of the system. If `rule_id`
is incorrect then you have to delete the rule completely and re-initialize it with the
correct `rule_id`
First add all the examples from queries like so:
```sh
./post_rule.sh ./rules/queries/*.json
```
Then to selectively update a rule add the file of your choosing to update:
```sh
./update_rule.sh ./rules/updates/<filename>.json
```
Take note that the ones with "id" must be changed to a GUID that only you know about through
a `./find_rules.sh`. For example to grab a GUID id off of the first found record that exists
you can do: `./find_rules.sh | jq '.data[0].id'` and then replace the id in `updates/simplest_update_risk_score_by_id.json` with that particular id to watch it happen.

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/disable_rule.json Normal file
View file

@ -0,0 +1,4 @@
{
"rule_id": "query-rule-id",
"enabled": false
}

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/enabled_rule.json Normal file
View file

@ -0,0 +1,4 @@
{
"rule_id": "query-rule-id",
"enabled": true
}

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/simplest_update_risk_score_by_id.json Normal file
View file

@ -0,0 +1,4 @@
{
"id": "ade31ba8-dc49-4c18-b7f4-370b35df5f57",
"risk_score": 38
}

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/simplest_update_risk_score_by_rule_id.json Normal file
View file

@ -0,0 +1,4 @@
{
"rule_id": "query-rule-id",
"risk_score": 98
}

4
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/simplest_updated_name.json Normal file
View file

@ -0,0 +1,4 @@
{
"rule_id": "query-rule-id",
"name": "Changes only the name to this new value"
}

80
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/updates/update_query_everything.json Normal file
View file

@ -0,0 +1,80 @@
{
"name": "Updates a query with all possible fields that can be updated",
"description": "Kitchen Sink (everything) query that has all possible fields filled out.",
"false_positives": [
"https://www.example.com/some-article-about-a-false-positive",
"some text string about why another condition could be a false positive"
],
"rule_id": "rule-id-everything",
"filters": [
{
"query": {
"match_phrase": {
"host.name": "siem-windows"
}
}
},
{
"exists": {
"field": "host.hostname"
}
}
],
"enabled": false,
"immutable": true,
"index": ["auditbeat-*", "filebeat-*"],
"interval": "5m",
"query": "user.name: root or user.name: admin",
"output_index": ".siem-signals-default",
"meta": {
"anything_you_want_ui_related_or_otherwise": {
"as_deep_structured_as_you_need": {
"any_data_type": {}
}
}
},
"language": "kuery",
"risk_score": 1,
"max_signals": 100,
"tags": ["tag 1", "tag 2", "any tag you want"],
"to": "now",
"from": "now-6m",
"severity": "high",
"type": "query",
"threats": [
{
"framework": "MITRE ATT&CK",
"tactic": {
"id": "TA0040",
"name": "impact",
"reference": "https://attack.mitre.org/tactics/TA0040/"
},
"techniques": [
{
"id": "T1499",
"name": "endpoint denial of service",
"reference": "https://attack.mitre.org/techniques/T1499/"
}
]
},
{
"framework": "Some other Framework you want",
"tactic": {
"id": "some-other-id",
"name": "Some other name",
"reference": "https://example.com"
},
"techniques": [
{
"id": "some-other-id",
"name": "some other technique name",
"reference": "https://example.com"
}
]
}
],
"references": [
"http://www.example.com/some-article-about-attack",
"Some plain text string here explaining why this is a valid thing to look out for"
]
}

13
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/rules/watch_longmont.json
View file

@ -1,13 +0,0 @@
{
"rule_id": "rule-longmont",
"risk_score": 5,
"description": "Detect Longmont activity",
"interval": "24h",
"name": "Detect Longmont activity",
"severity": "high",
"type": "query",
"from": "now-1y",
"to": "now",
"query": "user.name: root or user.name: admin",
"language": "kuery"
}

6
x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts/update_rule.sh
View file

@ -10,11 +10,11 @@ set -e
./check_env_variables.sh
# Uses a default if no argument is specified
RULES=(${@:-./rules/root_or_admin_update_1.json})
RULES=(${@:-./rules/updates/simplest_updated_name.json})
# Example: ./update_rule.sh
# Example: ./update_rule.sh ./rules/root_or_admin_1.json
# Example glob: ./post_rule.sh ./rules/*
# Example: ./update_rule.sh ./rules/updates/simplest_updated_name.json
# Example glob: ./post_rule.sh ./rules/updates/*
for RULE in "${RULES[@]}"
do {
[ -e "$RULE" ] || continue