[ML] Adds auditbeat process data recognizer modules (#25716)

* [ML] Adds auditbeat process data recognizer modules

* [ML] Sorts Kibana objects by title in recognizer job wizard

* [ML] Rename auditbeat modules Kibana objects to snake_case

* [ML] Remove auditbeat docker module kibana files

* [ML] Add auditbeat docker kibana objects with lowercase names

* [ML] Remove auditbeat host module kibana files

* [ML] Add auditbeat host module files with lowercase filenames
This commit is contained in:
Pete Harverson 2018-11-16 15:33:42 +00:00 committed by GitHub
parent 8e21a1b426
commit b52ddd206a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 660 additions and 1 deletions

View file

@ -225,7 +225,7 @@
<div class="row charts-container" ng-repeat='(key, value) in formConfig.kibanaObjects'>
<h4 class="euiTitle euiTitle--small">{{ui.kibanaLabels[key]}}</h4>
<div class='save-objects-list'>
<div ng-repeat='obj in value' class='job-container'>
<div ng-repeat='obj in value | orderBy:"title"' class='job-container'>
<div class='labels'>
<div class='title' ng-class="{exists: obj.exists}">
{{obj.title}}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Audit Events",
"description": "All events occurring within docker containers",
"panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":13,\"h\":13,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_container_count\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":13,\"y\":0,\"w\":35,\"h\":13,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_container_images\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":13,\"w\":48,\"h\":13,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_container_event_volume\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":26,\"w\":24,\"h\":15,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_processes\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":26,\"w\":24,\"h\":15,\"i\":\"5\"},\"version\":\"6.4.0\",\"panelIndex\":\"5\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_process_presence\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":41,\"w\":24,\"h\":15,\"i\":\"6\"},\"version\":\"6.4.0\",\"panelIndex\":\"6\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_docker_commands\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":41,\"w\":24,\"h\":15,\"i\":\"7\"},\"version\":\"6.4.0\",\"panelIndex\":\"7\",\"type\":\"search\",\"id\":\"ml_auditbeat_docker_events\",\"embeddableConfig\":{}}]",
"optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,16 @@
{
"title": "ML Auditbeat Docker: Docker Events",
"description": "Audit Events Correlated with Docker Metadata",
"hits": 0,
"columns": [
"_source"
],
"sort": [
"@timestamp",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"docker.container.id\",\"value\":\"exists\"},\"exists\":{\"field\":\"docker.container.id\"},\"$state\":{\"store\":\"appState\"}}]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Commands",
"visState": "{\"title\":\"ML Auditbeat Docker: Commands\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"process.title\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Container Count",
"visState": "{\"title\":\"ML Auditbeat Docker: Container Count\",\"type\":\"metric\",\"params\":{\"addTooltip\":true,\"addLegend\":false,\"type\":\"metric\",\"metric\":{\"percentageMode\":false,\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"metricColorMode\":\"None\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"labels\":{\"show\":true},\"invertColors\":false,\"style\":{\"bgFill\":\"#000\",\"bgColor\":false,\"labelColor\":false,\"subText\":\"\",\"fontSize\":60}}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"docker.container.id\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Container Event Volume",
"visState": "{\"title\":\"ML Auditbeat Docker: Container Event Volume\",\"type\":\"line\",\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"docker.container.id\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Container Images",
"visState": "{\"title\":\"ML Auditbeat Docker: Container Images\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"docker.container.image\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,12 @@
{
"title": "ML Auditbeat Docker: Process Presence",
"visState": "{\"title\":\"ML Auditbeat Docker: Process Presence\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Unique\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"process.exe\",\"customLabel\":\"Unique\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"process.exe\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"docker.container.name\",\"size\":1,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"row\":true}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Docker: Processes",
"visState": "{\"title\":\"ML Auditbeat Docker: Processes\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"process.exe\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"docker.container.name\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"row\":true}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_docker_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,5 @@
{
"src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgAgMAAAAOFJJnAAAADFBMVEUAAAAAAAABf3X////ZaOWRAAAAAXRSTlMAQObYZgAAAAFiS0dEAxEMTPIAAAAfSURBVBjTYwgNDXVqBBIMcEYAAwNTAwMD60hkYIQGAIQRIolX2EV0AAAAAElFTkSuQmCC",
"height": 32,
"width": 32
}

View file

@ -0,0 +1,86 @@
{
"id": "auditbeat_process_docker",
"title": "Auditbeat Docker processes",
"description": "Detect unusual processes on Docker containers",
"type": "Auditbeat data",
"logoFile": "logo.json",
"defaultIndexPattern": "auditbeat-*",
"query": {
"bool": {
"must": [
{
"exists": {
"field": "auditd"
}
},
{
"exists": {
"field": "docker.container.id"
}
}
]
}
},
"jobs": [
{
"id": "docker_high_count_events",
"file": "docker_high_count_events.json"
},
{
"id": "docker_suspicious_process_activity",
"file": "docker_suspicious_process_activity.json"
}
],
"datafeeds": [
{
"id": "datafeed-docker_high_count_events",
"file": "datafeed_docker_high_count_events.json",
"job_id": "docker_high_count_events"
},
{
"id": "datafeed-docker_suspicious_process_activity",
"file": "datafeed_docker_suspicious_process_activity.json",
"job_id": "docker_suspicious_process_activity"
}
],
"kibana": {
"dashboard": [
{
"id": "ml_auditbeat_docker_audit_events",
"file": "ml_auditbeat_docker_audit_events.json"
}
],
"search": [
{
"id": "ml_auditbeat_docker_events",
"file": "ml_auditbeat_docker_events.json"
}
],
"visualization": [
{
"id": "ml_auditbeat_docker_commands",
"file": "ml_auditbeat_docker_commands.json"
},
{
"id": "ml_auditbeat_docker_container_count",
"file": "ml_auditbeat_docker_container_count.json"
},
{
"id": "ml_auditbeat_docker_container_event_volume",
"file": "ml_auditbeat_docker_container_event_volume.json"
},
{
"id": "ml_auditbeat_docker_container_images",
"file": "ml_auditbeat_docker_container_images.json"
},
{
"id": "ml_auditbeat_docker_processes",
"file": "ml_auditbeat_docker_processes.json"
},
{
"id": "ml_auditbeat_docker_process_presence",
"file": "ml_auditbeat_docker_process_presence.json"
}
]
}
}

View file

@ -0,0 +1,27 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"types": [],
"query": {
"bool": {
"must": [
{
"match": {
"event.type": "syscall"
}
},
{
"exists": {
"field":"docker.container.id"
}
}
]
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "auto"
}
}

View file

@ -0,0 +1,27 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"types": [],
"query": {
"bool": {
"must": [
{
"match": {
"event.type": "syscall"
}
},
{
"exists": {
"field":"docker.container.id"
}
}
]
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "auto"
}
}

View file

@ -0,0 +1,35 @@
{
"job_type": "anomaly_detector",
"description": "Auditbeat: Detect Unusual Increases in Docker Process Volume",
"groups": ["auditbeat"],
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
"detector_description": "high_count partitionfield=\"docker.container.id\"",
"function": "high_count",
"partition_field_name": "docker.container.id"
}
],
"influencers": [
"process.exe"
]
},
"analysis_limits": {
"model_memory_limit": "256mb",
"categorization_examples_limit": 4
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Docker Events",
"time_range": "1h",
"url_value": "kibana#/dashboard/ml_auditbeat_docker_audit_events?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:lucene,query:'docker.container.id:\"$docker.container.id$\"'))"
}
]
}
}

View file

@ -0,0 +1,35 @@
{
"job_type": "anomaly_detector",
"description": "Auditbeat: Detect Rare Process Executions in Docker Containers",
"groups": ["auditbeat"],
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
"detector_description": "rare by 'process.exe'",
"function": "rare",
"by_field_name": "process.exe"
}
],
"influencers": [
"process.exe",
"docker.container.id"
]
},
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Docker Events",
"time_range": "1h",
"url_value": "kibana#/dashboard/ml_auditbeat_docker_audit_events?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:lucene,query:'docker.container.id:\"$docker.container.id$\" AND process.exe:\"$process.exe$\"'))"
}
]
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Audit Events",
"description": "All events occuring directly on host machines",
"panelsJSON": "[{\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":12,\"i\":\"1\"},\"version\":\"6.4.0\",\"panelIndex\":\"1\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_event_volume\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":12,\"w\":24,\"h\":15,\"i\":\"2\"},\"version\":\"6.4.0\",\"panelIndex\":\"2\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_kernel_actions\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":12,\"w\":24,\"h\":15,\"i\":\"3\"},\"version\":\"6.4.0\",\"panelIndex\":\"3\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_kernel_action_presence\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":27,\"w\":24,\"h\":15,\"i\":\"4\"},\"version\":\"6.4.0\",\"panelIndex\":\"4\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_processes\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":27,\"w\":24,\"h\":15,\"i\":\"5\"},\"version\":\"6.4.0\",\"panelIndex\":\"5\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_process_presence\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":42,\"w\":24,\"h\":15,\"i\":\"6\"},\"version\":\"6.4.0\",\"panelIndex\":\"6\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_command_line\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":42,\"w\":24,\"h\":15,\"i\":\"7\"},\"version\":\"6.4.0\",\"panelIndex\":\"7\",\"type\":\"visualization\",\"id\":\"ml_auditbeat_hosts_exe_thing\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":0,\"y\":57,\"w\":24,\"h\":15,\"i\":\"8\"},\"version\":\"6.4.0\",\"panelIndex\":\"8\",\"type\":\"search\",\"id\":\"ml_auditbeat_hosts_events\",\"embeddableConfig\":{}},{\"gridData\":{\"x\":24,\"y\":57,\"w\":24,\"h\":15,\"i\":\"9\"},\"version\":\"6.4.0\",\"panelIndex\":\"9\",\"type\":\"search\",\"id\":\"ml_auditbeat_all_events\",\"embeddableConfig\":{}}]",
"optionsJSON": "{\"darkTheme\":false,\"useMargins\":true,\"hidePanelTitles\":false}",
"version": 1,
"timeRestore": false,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,16 @@
{
"title": "ML Auditbeat: All Events",
"description": "All Audit Events Captured By Auditbeat",
"hits": 0,
"columns": [
"_source"
],
"sort": [
"@timestamp",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,16 @@
{
"title": "ML Auditbeat Hosts: Host Events",
"description": "Audit Events occurring directly on host machines",
"hits": 0,
"columns": [
"_source"
],
"sort": [
"@timestamp",
"desc"
],
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"index\":\"INDEX_PATTERN_ID\",\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[{\"meta\":{\"index\":\"INDEX_PATTERN_ID\",\"negate\":true,\"disabled\":false,\"alias\":null,\"type\":\"exists\",\"key\":\"docker.container.id\",\"value\":\"exists\"},\"exists\":{\"field\":\"docker.container.id\"},\"$state\":{\"store\":\"appState\"}}]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Command Line",
"visState": "{\"title\":\"ML Auditbeat Hosts: Command Line\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMetricsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"process.title\",\"size\":100,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Event Volume",
"visState": "{\"title\":\"ML Auditbeat Hosts: Event Volume\",\"type\":\"line\",\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"line\",\"mode\":\"normal\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"beat.hostname\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Exe Thing",
"visState": "{\"title\":\"ML Auditbeat Hosts: Exe Thing\",\"type\":\"pie\",\"params\":{\"type\":\"pie\",\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"isDonut\":true,\"labels\":{\"show\":false,\"values\":true,\"last_level\":true,\"truncate\":100}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"process.exe\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"auditd.summary.object.primary\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Kernel Action Presence",
"visState": "{\"title\":\"ML Auditbeat Hosts: Kernel Action Presence\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of event.action\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Unique count of event.action\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"event.action\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.action\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Kernel Actions",
"visState": "{\"title\":\"ML Auditbeat Hosts: Kernel Actions\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"event.action\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Process Presence",
"visState": "{\"title\":\"ML Auditbeat Hosts: Process Presence\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Unique count of process.exe\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Unique count of process.exe\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"process.exe\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,11 @@
{
"title": "ML Auditbeat Hosts: Processes",
"visState": "{\"title\":\"ML Auditbeat Hosts: Processes\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"process.exe\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"}}]}",
"uiStateJSON": "{}",
"description": "",
"savedSearchId": "ml_auditbeat_hosts_events",
"version": 1,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"
}
}

View file

@ -0,0 +1,5 @@
{
"src": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgAgMAAAAOFJJnAAAADFBMVEUAAAAAAAABf3X////ZaOWRAAAAAXRSTlMAQObYZgAAAAFiS0dEAxEMTPIAAAAfSURBVBjTYwgNDXVqBBIMcEYAAwNTAwMD60hkYIQGAIQRIolX2EV0AAAAAElFTkSuQmCC",
"height": 32,
"width": 32
}

View file

@ -0,0 +1,96 @@
{
"id": "auditbeat_process_hosts",
"title": "Auditbeat host processes",
"description": "Detect unusual processes on hosts",
"type": "Auditbeat data",
"logoFile": "logo.json",
"defaultIndexPattern": "auditbeat-*",
"query": {
"bool": {
"must": [
{
"exists": {
"field": "auditd"
}
}
],
"must_not": [
{
"exists": {
"field": "docker.container.id"
}
}
]
}
},
"jobs": [
{
"id": "hosts_high_count_events",
"file": "hosts_high_count_events.json"
},
{
"id": "hosts_suspicious_process_activity",
"file": "hosts_suspicious_process_activity.json"
}
],
"datafeeds": [
{
"id": "datafeed-hosts_high_count_events",
"file": "datafeed_hosts_high_count_events.json",
"job_id": "hosts_high_count_events"
},
{
"id": "datafeed-hosts_suspicious_process_activity",
"file": "datafeed_hosts_suspicious_process_activity.json",
"job_id": "hosts_suspicious_process_activity"
}
],
"kibana": {
"dashboard": [
{
"id": "ml_auditbeat_hosts_audit_events",
"file": "ml_auditbeat_hosts_audit_events.json"
}
],
"search": [
{
"id": "ml_auditbeat_hosts_events",
"file": "ml_auditbeat_hosts_events.json"
},
{
"id": "ml_auditbeat_all_events",
"file": "ml_auditbeat_all_events.json"
}
],
"visualization": [
{
"id": "ml_auditbeat_hosts_command_line",
"file": "ml_auditbeat_hosts_command_line.json"
},
{
"id": "ml_auditbeat_hosts_event_volume",
"file": "ml_auditbeat_hosts_event_volume.json"
},
{
"id": "ml_auditbeat_hosts_exe_thing",
"file": "ml_auditbeat_hosts_exe_thing.json"
},
{
"id": "ml_auditbeat_hosts_kernel_action_presence",
"file": "ml_auditbeat_hosts_kernel_action_presence.json"
},
{
"id": "ml_auditbeat_hosts_kernel_actions",
"file": "ml_auditbeat_hosts_kernel_actions.json"
},
{
"id": "ml_auditbeat_hosts_process_presence",
"file": "ml_auditbeat_hosts_process_presence.json"
},
{
"id": "ml_auditbeat_hosts_processes",
"file": "ml_auditbeat_hosts_processes.json"
}
]
}
}

View file

@ -0,0 +1,29 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"types": [],
"query":{
"bool": {
"must":[
{
"match": {
"event.type": "syscall"
}
}
],
"must_not": [
{
"exists": {
"field": "docker.container.id"
}
}
]
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "auto"
}
}

View file

@ -0,0 +1,29 @@
{
"job_id": "JOB_ID",
"indexes": [
"INDEX_PATTERN_NAME"
],
"types": [],
"query":{
"bool": {
"must":[
{
"match": {
"event.type": "syscall"
}
}
],
"must_not": [
{
"exists": {
"field": "docker.container.id"
}
}
]
}
},
"scroll_size": 1000,
"chunking_config": {
"mode": "auto"
}
}

View file

@ -0,0 +1,36 @@
{
"job_type": "anomaly_detector",
"description": "Auditbeat Hosts: Detect Unusual Increases in Host Process Volume",
"groups": ["auditbeat"],
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
"detector_description": "high_count partitionfield=\"beat.hostname\"",
"function": "high_count",
"partition_field_name": "beat.hostname"
}
],
"influencers": [
"beat.hostname",
"process.exe"
]
},
"analysis_limits": {
"model_memory_limit": "256mb",
"categorization_examples_limit": 4
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Host Events",
"time_range": "1h",
"url_value": "kibana#/dashboard/ml_auditbeat_hosts_audit_events?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:lucene,query:'beat.hostname:\"$beat.hostname$\"'))"
}
]
}
}

View file

@ -0,0 +1,35 @@
{
"job_type": "anomaly_detector",
"description": "Auditbeat Hosts: Detect Rare Process Executions on Hosts",
"groups": ["auditbeat"],
"analysis_config": {
"bucket_span": "1h",
"detectors": [
{
"detector_description": "rare by 'process.exe'",
"function": "rare",
"by_field_name": "process.exe"
}
],
"influencers": [
"process.exe",
"beat.hostname"
]
},
"analysis_limits": {
"model_memory_limit": "256mb"
},
"data_description": {
"time_field": "@timestamp",
"time_format": "epoch_ms"
},
"custom_settings": {
"custom_urls": [
{
"url_name": "Host Events",
"time_range": "1h",
"url_value": "kibana#/dashboard/ml_auditbeat_hosts_audit_events?_g=(time:(from:'$earliest$',mode:absolute,to:'$latest$'))&_a=(filters:!(),query:(language:lucene,query:'beat.hostname:\"$beat.hostname$\" AND process.exe:\"$process.exe$\"'))"
}
]
}
}