maxmustermann/kibana
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[ML] Adds SIEM auditbeat and metricbeat data recognizer modules (#39678) (#39789)

Browse source 
* [ML] Adds SIEM auditbeat and metricbeat data recognizer modules

* [ML] Added agent.type clause to winlogbeat manifest query

* [ML] Remove mistakenly included packetbeat module, plus text edits
This commit is contained in:
Pete Harverson 2019-06-27 16:43:22 +01:00 committed by GitHub
parent 2a6100e9c2
commit e268bfc035
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 256 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js
View file

@ -21,6 +21,8 @@ describe('ML - data recognizer', () => {
'nginx_ecs',
'sample_data_ecommerce',
'sample_data_weblogs',
'siem_auditbeat_ecs',
'siem_winlogbeat_ecs',
];
// check all module IDs are the same as the list above

3
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/logo.json Normal file
View file

@ -0,0 +1,3 @@
{
"icon": "securityAnalyticsApp"
}

46
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/manifest.json Normal file
View file

@ -0,0 +1,46 @@
{
"id": "siem_auditbeat_ecs",
"title": "SIEM Auditbeat",
"description": "Detect suspicious logins and unusual processes in Auditbeat ECS data (beta)",
"type": "Auditbeat data",
"logoFile": "logo.json",
"defaultIndexPattern": "auditbeat-*",
"query": {
"bool" : {
"filter": [
{
"bool": {
"should" : [
{ "terms": { "event.action": [ "process_started", "executed" ] } },
{ "term": { "event.category": "authentication" }}
],
"minimum_should_match" : 1
}
},
{ "term": { "agent.type": "auditbeat" } }
]
}
},
"jobs": [
{
"id": "rare_process_linux_ecs",
"file": "rare_process_linux_ecs.json"
},
{
"id": "suspicious_login_activity_ecs",
"file": "suspicious_login_activity_ecs.json"
}
],
"datafeeds": [
{
"id": "datafeed-rare_process_linux_ecs",
"file": "datafeed_rare_process_linux_ecs.json",
"job_id": "rare_process_linux_ecs"
},
{
"id": "datafeed-suspicious_login_activity_ecs",
"file": "datafeed_suspicious_login_activity_ecs.json",
"job_id": "suspicious_login_activity_ecs"
}
]
}

13
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/ml/datafeed_rare_process_linux_ecs.json Normal file
View file

@ -0,0 +1,13 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": [
{ "terms": { "event.action": [ "process_started", "executed" ] } }
]
}
}
}

13
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/ml/datafeed_suspicious_login_activity_ecs.json Normal file
View file

@ -0,0 +1,13 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": {
"term": { "event.category": "authentication" }
}
}
}
}

49
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/ml/rare_process_linux_ecs.json Normal file
View file

@ -0,0 +1,49 @@
{
"job_type": "anomaly_detector",
"description": "SIEM Auditbeat: Detect unusually rare processes on Linux (beta)",
"groups": ["siem"],
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "rare process executions on Linux",
"function": "rare",
"by_field_name": "process.name",
"partition_field_name": "host.name"
}
],
"influencers": [
"host.name",
"process.name",
"user.name"
]
},
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"created_by": "ml-module-siem-auditbeat",
"custom_urls": [
{
"url_name": "Host Details by process name",
"url_value": "siem#/hosts/$host.name$?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22$process.name$%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Host Details by user name",
"url_value": "siem#/hosts/$host.name$?_g=()&kqlQuery=(filterQuery:(expression:'user.name%20:%20%22$user.name$%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Hosts Overview by process name",
"url_value": "siem#/hosts?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22$process.name$%22',kind:kuery),queryLocation:hosts.page,type:page)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Hosts Overview by user name",
"url_value": "siem#/hosts?_g=()&kqlQuery=(filterQuery:(expression:'user.name%20:%20%22$user.name$%22',kind:kuery),queryLocation:hosts.page,type:page)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
}
]
}
}

36
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_auditbeat_ecs/ml/suspicious_login_activity_ecs.json Normal file
View file

@ -0,0 +1,36 @@
{
"job_type": "anomaly_detector",
"description": "SIEM Auditbeat: Detect unusually high number of authentication attempts (beta)",
"groups": ["siem"],
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "high number of authentication attempts",
"function": "high_non_zero_count",
"partition_field_name": "host.name"
}
],
"influencers": [
"host.name",
"user.name",
"source.ip"
]
},
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"created_by": "ml-module-siem-auditbeat",
"custom_urls": [
{
"url_name": "IP Address Details",
"url_value": "siem#/network/ip/$source.ip$?_g=()&kqlQuery=(filterQuery:!n,queryLocation:network.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
}
]
}
}

3
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat_ecs/logo.json Normal file
View file

@ -0,0 +1,3 @@
{
"icon": "securityAnalyticsApp"
}

29
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat_ecs/manifest.json Normal file
View file

@ -0,0 +1,29 @@
{
"id": "siem_winlogbeat_ecs",
"title": "SIEM Winlogbeat",
"description": "Detect unusual processes in Winlogbeat ECS data (beta)",
"type": "Winlogbeat data",
"logoFile": "logo.json",
"defaultIndexPattern": "winlogbeat-*",
"query": {
"bool": {
"filter": [
{ "term": { "event.action": "Process Create (rule: ProcessCreate)" } },
{ "term": { "agent.type": "winlogbeat" } }
]
}
},
"jobs": [
{
"id": "rare_process_windows_ecs",
"file": "rare_process_windows_ecs.json"
}
],
"datafeeds": [
{
"id": "datafeed-rare_process_windows_ecs",
"file": "datafeed_rare_process_windows_ecs.json",
"job_id": "rare_process_windows_ecs"
}
]
}

13
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat_ecs/ml/datafeed_rare_process_windows_ecs.json Normal file
View file

@ -0,0 +1,13 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"query": {
"bool": {
"filter": {
"term": { "event.action": "Process Create (rule: ProcessCreate)" }
}
}
}
}

49
x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/siem_winlogbeat_ecs/ml/rare_process_windows_ecs.json Normal file
View file

@ -0,0 +1,49 @@
{
"job_type": "anomaly_detector",
"description": "SIEM Winlogbeat: Detect unusually rare processes on Windows (beta)",
"groups": ["siem"],
"analysis_config": {
"bucket_span": "15m",
"detectors": [
{
"detector_description": "rare process executions on Windows",
"function": "rare",
"by_field_name": "process.name",
"partition_field_name": "host.name"
}
],
"influencers": [
"host.name",
"process.name",
"user.name"
]
},
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"created_by": "ml-module-siem-winlogbeat",
"custom_urls": [
{
"url_name": "Host Details by process name",
"url_value": "siem#/hosts/$host.name$?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22$process.name$%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Host Details by user name",
"url_value": "siem#/hosts/$host.name$?_g=()&kqlQuery=(filterQuery:(expression:'user.name%20:%20%22$user.name$%22',kind:kuery),queryLocation:hosts.details,type:details)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Hosts Overview by process name",
"url_value": "siem#/hosts?_g=()&kqlQuery=(filterQuery:(expression:'process.name%20:%20%22$process.name$%22',kind:kuery),queryLocation:hosts.page,type:page)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
},
{
"url_name": "Hosts Overview by user name",
"url_value": "siem#/hosts?_g=()&kqlQuery=(filterQuery:(expression:'user.name%20:%20%22$user.name$%22',kind:kuery),queryLocation:hosts.page,type:page)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))"
}
]
}
}