From 313d85e985141f5727da0f55214ab8db57d665d9 Mon Sep 17 00:00:00 2001 From: Craig Chamberlain Date: Thu, 10 Dec 2020 14:02:41 -0500 Subject: [PATCH] [ML] Adds security_linux and security_windows Modules (#85065) * initial commit refactored multi-index, multi-pipeline jobs for 7.11. These are new modules that will live alongside the existing jobs. * Update ml_modules.tsx added new module names to the list * Update get_module.ts added new module names * Linter fixes * Order matters * manifest fixes added colon char to the module name and shortened the description * additon to description after talking with the security team today, adding this suggested text to the beginning of the description so it will tend to be visible to the user: "This is a new refactored job which works on ECS compatible events across multiple indices." * Adjust module recognizer test for auditbeat dataset * influencers changes to the metadata jobs to make influencers identical to the originals * change for security app changes to two datafeeds needed for logic in the Security app - added the suffix "_ecs" to two ids. Co-authored-by: Garrett Spong Co-authored-by: Robert Oskamp --- .../modules/security_linux/logo.json | 3 + .../modules/security_linux/manifest.json | 104 ++++++++++++++++ ...x_anomalous_network_port_activity_ecs.json | 76 ++++++++++++ ...linux_anomalous_process_all_hosts_ecs.json | 101 ++++++++++++++++ ...feed_v2_linux_anomalous_user_name_ecs.json | 71 +++++++++++ ...tafeed_v2_linux_rare_metadata_process.json | 66 +++++++++++ .../datafeed_v2_linux_rare_metadata_user.json | 66 +++++++++++ ...eed_v2_rare_process_by_host_linux_ecs.json | 71 +++++++++++ ...x_anomalous_network_port_activity_ecs.json | 55 +++++++++ ...linux_anomalous_process_all_hosts_ecs.json | 54 +++++++++ .../ml/v2_linux_anomalous_user_name_ecs.json | 54 +++++++++ .../ml/v2_linux_rare_metadata_process.json | 36 ++++++ .../ml/v2_linux_rare_metadata_user.json | 35 ++++++ .../ml/v2_rare_process_by_host_linux_ecs.json | 55 +++++++++ .../modules/security_windows/logo.json | 3 + .../modules/security_windows/manifest.json | 112 ++++++++++++++++++ ...d_v2_rare_process_by_host_windows_ecs.json | 47 ++++++++ ...indows_anomalous_network_activity_ecs.json | 71 +++++++++++ ...2_windows_anomalous_path_activity_ecs.json | 47 ++++++++ ...ndows_anomalous_process_all_hosts_ecs.json | 47 ++++++++ ...v2_windows_anomalous_process_creation.json | 47 ++++++++ ...ed_v2_windows_anomalous_user_name_ecs.json | 47 ++++++++ ...feed_v2_windows_rare_metadata_process.json | 23 ++++ ...atafeed_v2_windows_rare_metadata_user.json | 23 ++++ .../v2_rare_process_by_host_windows_ecs.json | 57 +++++++++ ...indows_anomalous_network_activity_ecs.json | 56 +++++++++ ...2_windows_anomalous_path_activity_ecs.json | 54 +++++++++ ...ndows_anomalous_process_all_hosts_ecs.json | 56 +++++++++ ...v2_windows_anomalous_process_creation.json | 57 +++++++++ .../v2_windows_anomalous_user_name_ecs.json | 56 +++++++++ .../ml/v2_windows_rare_metadata_process.json | 38 ++++++ .../ml/v2_windows_rare_metadata_user.json | 37 ++++++ .../components/ml_popover/ml_modules.tsx | 2 + .../apis/ml/modules/get_module.ts | 2 + .../apis/ml/modules/recognize_module.ts | 2 +- 35 files changed, 1730 insertions(+), 1 deletion(-) create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/logo.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_network_port_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_process_all_hosts_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_user_name_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_rare_process_by_host_linux_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_network_port_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_process_all_hosts_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_user_name_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_rare_process_by_host_linux_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/logo.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/manifest.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_rare_process_by_host_windows_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_network_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_path_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_all_hosts_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_creation.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_user_name_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_user.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_rare_process_by_host_windows_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_network_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_path_activity_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_all_hosts_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_creation.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_user_name_ecs.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_process.json create mode 100644 x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_user.json diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/logo.json new file mode 100644 index 000000000000..862f970b7405 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "logoSecurity" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/manifest.json new file mode 100644 index 000000000000..c627e5a6f625 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/manifest.json @@ -0,0 +1,104 @@ +{ + "id": "security_linux", + "title": "Security: Linux", + "description": "Detect suspicious activity using ECS Linux events. Tested with Auditbeat and the Elastic agent.", + "type": "linux data", + "logoFile": "logo.json", + "defaultIndexPattern": "auditbeat-*,logs-endpoint.events.*", + "query": { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + } + ] + } + }, + "jobs": [ + { + "id": "v2_rare_process_by_host_linux_ecs", + "file": "v2_rare_process_by_host_linux_ecs.json" + }, + { + "id": "v2_linux_rare_metadata_user", + "file": "v2_linux_rare_metadata_user.json" + }, + { + "id": "v2_linux_rare_metadata_process", + "file": "v2_linux_rare_metadata_process.json" + }, + { + "id": "v2_linux_anomalous_user_name_ecs", + "file": "v2_linux_anomalous_user_name_ecs.json" + }, + { + "id": "v2_linux_anomalous_process_all_hosts_ecs", + "file": "v2_linux_anomalous_process_all_hosts_ecs.json" + }, + { + "id": "v2_linux_anomalous_network_port_activity_ecs", + "file": "v2_linux_anomalous_network_port_activity_ecs.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-v2_rare_process_by_host_linux_ecs", + "file": "datafeed_v2_rare_process_by_host_linux_ecs.json", + "job_id": "v2_rare_process_by_host_linux_ecs" + }, + { + "id": "datafeed-v2_linux_rare_metadata_user", + "file": "datafeed_v2_linux_rare_metadata_user.json", + "job_id": "v2_linux_rare_metadata_user" + }, + { + "id": "datafeed-v2_linux_rare_metadata_process", + "file": "datafeed_v2_linux_rare_metadata_process.json", + "job_id": "v2_linux_rare_metadata_process" + }, + { + "id": "datafeed-v2_linux_anomalous_user_name_ecs", + "file": "datafeed_v2_linux_anomalous_user_name_ecs.json", + "job_id": "v2_linux_anomalous_user_name_ecs" + }, + { + "id": "datafeed-v2_linux_anomalous_process_all_hosts_ecs", + "file": "datafeed_v2_linux_anomalous_process_all_hosts_ecs.json", + "job_id": "v2_linux_anomalous_process_all_hosts_ecs" + }, + { + "id": "datafeed-v2_linux_anomalous_network_port_activity_ecs", + "file": "datafeed_v2_linux_anomalous_network_port_activity_ecs.json", + "job_id": "v2_linux_anomalous_network_port_activity_ecs" + } + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_network_port_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_network_port_activity_ecs.json new file mode 100644 index 000000000000..67c198b3f56e --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_network_port_activity_ecs.json @@ -0,0 +1,76 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": + { + "filter": [ + {"term": {"event.category": "network"}}, + {"term": {"event.type": "start"}} + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ], + "must_not": [ + { + "bool": { + "should": [ + {"term": {"destination.ip": "127.0.0.1"}}, + {"term": {"destination.ip": "::"}}, + {"term": {"destination.ip": "::1"}}, + {"term": {"user.name":"jenkins"}} + ] + } + } + ] + } + } + } diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_process_all_hosts_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_process_all_hosts_ecs.json new file mode 100644 index 000000000000..da41aff66ea0 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_process_all_hosts_ecs.json @@ -0,0 +1,101 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ], + "must_not": [ + { + "bool": { + "should": [ + { + "term": { + "user.name": "jenkins-worker" + } + }, + { + "term": { + "user.name": "jenkins-user" + } + }, + { + "term": { + "user.name": "jenkins" + } + }, + { + "wildcard": { + "process.name": { + "wildcard": "jenkins*" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_user_name_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_user_name_ecs.json new file mode 100644 index 000000000000..673de388e68b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_anomalous_user_name_ecs.json @@ -0,0 +1,71 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_process.json new file mode 100644 index 000000000000..b79d97ef5e40 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_process.json @@ -0,0 +1,66 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "destination.ip": "169.254.169.254" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_user.json new file mode 100644 index 000000000000..b79d97ef5e40 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_linux_rare_metadata_user.json @@ -0,0 +1,66 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "destination.ip": "169.254.169.254" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_rare_process_by_host_linux_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_rare_process_by_host_linux_ecs.json new file mode 100644 index 000000000000..673de388e68b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/datafeed_v2_rare_process_by_host_linux_ecs.json @@ -0,0 +1,71 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.type": { + "query": "linux", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "debian", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "redhat", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "suse", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.family": { + "query": "ubuntu", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_network_port_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_network_port_activity_ecs.json new file mode 100644 index 000000000000..2d3be4593c5d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_network_port_activity_ecs.json @@ -0,0 +1,55 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity.", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "network" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"destination.port\"", + "function": "rare", + "by_field_name": "destination.port" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name", + "destination.ip" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_process_all_hosts_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_process_all_hosts_ecs.json new file mode 100644 index 000000000000..03837cd77a5c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_process_all_hosts_ecs.json @@ -0,0 +1,54 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for processes that are unusual to all Linux hosts. Such unusual processes may indicate unauthorized services, malware, or persistence mechanisms.", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "512mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_user_name_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_user_name_ecs.json new file mode 100644 index 000000000000..3bc5afa6ec8d --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_anomalous_user_name_ecs.json @@ -0,0 +1,54 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "process" + ], + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Rare and unusual users that are not normally active may indicate unauthorized changes or activity by an unauthorized user which may be credentialed access or lateral movement.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_process.json new file mode 100644 index 000000000000..c550378dad0b --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_process.json @@ -0,0 +1,36 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for anomalous access to the metadata service by an unusual process. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "host.name", + "user.name", + "process.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_user.json new file mode 100644 index 000000000000..66f35bdce12c --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_linux_rare_metadata_user.json @@ -0,0 +1,35 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for anomalous access to the metadata service by an unusual user. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_rare_process_by_host_linux_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_rare_process_by_host_linux_ecs.json new file mode 100644 index 000000000000..fa87be8efb01 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_linux/ml/v2_rare_process_by_host_linux_ecs.json @@ -0,0 +1,55 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Linux - Looks for processes that are unusual to a particular Linux host. Such unusual processes may indicate unauthorized services, malware, or persistence mechanisms.", + "groups": [ + "security", + "auditbeat", + "endpoint", + "linux", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare process executions on Linux", + "function": "rare", + "by_field_name": "process.name", + "partition_field_name": "host.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-linux", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/logo.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/logo.json new file mode 100644 index 000000000000..862f970b7405 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "logoSecurity" +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/manifest.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/manifest.json new file mode 100644 index 000000000000..e1bd6eacc688 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/manifest.json @@ -0,0 +1,112 @@ +{ + "id": "security_windows", + "title": "Security: Windows", + "description": "Detects suspicious activity using ECS Windows events. Tested with Winlogbeat and the Elastic agent.", + "type": "windows data", + "logoFile": "logo.json", + "defaultIndexPattern": "winlogbeat-*,logs-endpoint.events.*", + "query": { + "bool": { + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + }, + "jobs": [ + { + "id": "v2_rare_process_by_host_windows_ecs", + "file": "v2_rare_process_by_host_windows_ecs.json" + }, + { + "id": "v2_windows_anomalous_network_activity_ecs", + "file": "v2_windows_anomalous_network_activity_ecs.json" + }, + { + "id": "v2_windows_anomalous_path_activity_ecs", + "file": "v2_windows_anomalous_path_activity_ecs.json" + }, + { + "id": "v2_windows_anomalous_process_all_hosts_ecs", + "file": "v2_windows_anomalous_process_all_hosts_ecs.json" + }, + { + "id": "v2_windows_anomalous_process_creation", + "file": "v2_windows_anomalous_process_creation.json" + }, + { + "id": "v2_windows_anomalous_user_name_ecs", + "file": "v2_windows_anomalous_user_name_ecs.json" + }, + { + "id": "v2_windows_rare_metadata_process", + "file": "v2_windows_rare_metadata_process.json" + }, + { + "id": "v2_windows_rare_metadata_user", + "file": "v2_windows_rare_metadata_user.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-v2_rare_process_by_host_windows_ecs", + "file": "datafeed_v2_rare_process_by_host_windows_ecs.json", + "job_id": "v2_rare_process_by_host_windows_ecs" + }, + { + "id": "datafeed-v2_windows_anomalous_network_activity_ecs", + "file": "datafeed_v2_windows_anomalous_network_activity_ecs.json", + "job_id": "v2_windows_anomalous_network_activity_ecs" + }, + { + "id": "datafeed-v2_windows_anomalous_path_activity_ecs", + "file": "datafeed_v2_windows_anomalous_path_activity_ecs.json", + "job_id": "v2_windows_anomalous_path_activity_ecs" + }, + { + "id": "datafeed-v2_windows_anomalous_process_all_hosts_ecs", + "file": "datafeed_v2_windows_anomalous_process_all_hosts_ecs.json", + "job_id": "v2_windows_anomalous_process_all_hosts_ecs" + }, + { + "id": "datafeed-v2_windows_anomalous_process_creation", + "file": "datafeed_v2_windows_anomalous_process_creation.json", + "job_id": "v2_windows_anomalous_process_creation" + }, + { + "id": "datafeed-v2_windows_anomalous_user_name_ecs", + "file": "datafeed_v2_windows_anomalous_user_name_ecs.json", + "job_id": "v2_windows_anomalous_user_name_ecs" + }, + { + "id": "datafeed-v2_windows_rare_metadata_process", + "file": "datafeed_v2_windows_rare_metadata_process.json", + "job_id": "v2_windows_rare_metadata_process" + }, + { + "id": "datafeed-v2_windows_rare_metadata_user", + "file": "datafeed_v2_windows_rare_metadata_user.json", + "job_id": "v2_windows_rare_metadata_user" + } + ] +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_rare_process_by_host_windows_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_rare_process_by_host_windows_ecs.json new file mode 100644 index 000000000000..fd3c03b3a3e9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_rare_process_by_host_windows_ecs.json @@ -0,0 +1,47 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_network_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_network_activity_ecs.json new file mode 100644 index 000000000000..d085cfa38c65 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_network_activity_ecs.json @@ -0,0 +1,71 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "network" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ], + "must_not": [ + { + "bool": { + "should": [ + { + "term": { + "destination.ip": "127.0.0.1" + } + }, + { + "term": { + "destination.ip": "127.0.0.53" + } + }, + { + "term": { + "destination.ip": "::1" + } + } + ], + "minimum_should_match": 1 + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_path_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_path_activity_ecs.json new file mode 100644 index 000000000000..fd3c03b3a3e9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_path_activity_ecs.json @@ -0,0 +1,47 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_all_hosts_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_all_hosts_ecs.json new file mode 100644 index 000000000000..fd3c03b3a3e9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_all_hosts_ecs.json @@ -0,0 +1,47 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_creation.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_creation.json new file mode 100644 index 000000000000..fd3c03b3a3e9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_process_creation.json @@ -0,0 +1,47 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_user_name_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_user_name_ecs.json new file mode 100644 index 000000000000..fd3c03b3a3e9 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_anomalous_user_name_ecs.json @@ -0,0 +1,47 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "event.category": "process" + } + }, + { + "term": { + "event.type": "start" + } + } + ], + "must": [ + { + "bool": { + "should": [ + { + "match": { + "host.os.family": { + "query": "windows", + "operator": "OR" + } + } + }, + { + "match": { + "host.os.type": { + "query": "windows", + "operator": "OR" + } + } + } + ] + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_process.json new file mode 100644 index 000000000000..f0be23df84c4 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_process.json @@ -0,0 +1,23 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "host.os.family": "windows" + } + }, + { + "term": { + "destination.ip": "169.254.169.254" + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_user.json new file mode 100644 index 000000000000..f0be23df84c4 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/datafeed_v2_windows_rare_metadata_user.json @@ -0,0 +1,23 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": [ + { + "term": { + "host.os.family": "windows" + } + }, + { + "term": { + "destination.ip": "169.254.169.254" + } + } + ] + } + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_rare_process_by_host_windows_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_rare_process_by_host_windows_ecs.json new file mode 100644 index 000000000000..a645d3167c30 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_rare_process_by_host_windows_ecs.json @@ -0,0 +1,57 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Detects unusually rare processes on Windows hosts.", + "groups": [ + "security", + "endpoint", + "event-log", + "sysmon", + "windows", + "winlogbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare process executions on Windows", + "function": "rare", + "by_field_name": "process.name", + "partition_field_name": "host.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_network_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_network_activity_ecs.json new file mode 100644 index 000000000000..61bafc605707 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_network_activity_ecs.json @@ -0,0 +1,56 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity.", + "groups": [ + "security", + "endpoint", + "sysmon", + "windows", + "winlogbeat", + "network" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name", + "destination.ip" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "64mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_path_activity_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_path_activity_ecs.json new file mode 100644 index 000000000000..9aea3305cc64 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_path_activity_ecs.json @@ -0,0 +1,54 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "security", + "sysmon", + "windows", + "winlogbeat", + "process" + ], + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for activity in unusual paths that may indicate execution of malware or persistence mechanisms. Windows payloads often execute from user profile paths.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.working_directory\"", + "function": "rare", + "by_field_name": "process.working_directory" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_all_hosts_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_all_hosts_ecs.json new file mode 100644 index 000000000000..07e8e872b1b8 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_all_hosts_ecs.json @@ -0,0 +1,56 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for processes that are unusual to all Windows hosts. Such unusual processes may indicate execution of unauthorized services, malware, or persistence mechanisms.", + "groups": [ + "security", + "endpoint", + "event-log", + "sysmon", + "windows", + "winlogbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.executable\"", + "function": "rare", + "by_field_name": "process.executable" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_creation.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_creation.json new file mode 100644 index 000000000000..e59d887ccc90 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_process_creation.json @@ -0,0 +1,57 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "security", + "endpoint", + "event-log", + "sysmon", + "windows", + "winlogbeat", + "process" + ], + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for unusual process relationships which may indicate execution of malware or persistence mechanisms.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "Unusual process creation activity", + "function": "rare", + "by_field_name": "process.name", + "partition_field_name": "process.parent.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_user_name_ecs.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_user_name_ecs.json new file mode 100644 index 000000000000..af04625e56fc --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_anomalous_user_name_ecs.json @@ -0,0 +1,56 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Rare and unusual users that are not normally active may indicate unauthorized changes or activity by an unauthorized user which may be credentialed access or lateral movement.", + "groups": [ + "security", + "endpoint", + "event-log", + "sysmon", + "windows", + "winlogbeat", + "process" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "process.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows", + "custom_urls": [ + { + "url_name": "Host Details by process name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Host Details by user name", + "url_value": "security/hosts/ml-hosts/$host.name$?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by process name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'process.name%20:%20%22$process.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + }, + { + "url_name": "Hosts Overview by user name", + "url_value": "security/hosts/ml-hosts?_g=()&query=(query:'user.name%20:%20%22$user.name$%22',language:kuery)&timerange=(global:(linkTo:!(timeline),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')),timeline:(linkTo:!(global),timerange:(from:'$earliest$',kind:absolute,to:'$latest$')))" + } + ] + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_process.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_process.json new file mode 100644 index 000000000000..e8f5317be030 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_process.json @@ -0,0 +1,38 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for anomalous access to the metadata service by an unusual process. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "endpoint", + "event-log", + "process", + "sysmon", + "windows", + "winlogbeat" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"process.name\"", + "function": "rare", + "by_field_name": "process.name" + } + ], + "influencers": [ + "process.name", + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows" + } +} diff --git a/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_user.json b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_user.json new file mode 100644 index 000000000000..027dbd84de33 --- /dev/null +++ b/x-pack/plugins/ml/server/models/data_recognizer/modules/security_windows/ml/v2_windows_rare_metadata_user.json @@ -0,0 +1,37 @@ +{ + "job_type": "anomaly_detector", + "description": "This is a new refactored job which works on ECS compatible events across multiple indices. Security: Windows - Looks for anomalous access to the metadata service by an unusual user. The metadata service may be targeted in order to harvest credentials or user data scripts containing secrets.", + "groups": [ + "security", + "endpoint", + "event-log", + "process", + "sysmon", + "windows", + "winlogbeat" + ], + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "rare by \"user.name\"", + "function": "rare", + "by_field_name": "user.name" + } + ], + "influencers": [ + "host.name", + "user.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-security-windows" + } +} diff --git a/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx b/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx index 4dccba08590a..f2f802124d72 100644 --- a/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx +++ b/x-pack/plugins/security_solution/public/common/components/ml_popover/ml_modules.tsx @@ -16,4 +16,6 @@ export const mlModules: string[] = [ 'siem_packetbeat', 'siem_winlogbeat', 'siem_winlogbeat_auth', + 'security_linux', + 'security_windows', ]; diff --git a/x-pack/test/api_integration/apis/ml/modules/get_module.ts b/x-pack/test/api_integration/apis/ml/modules/get_module.ts index 6c7cb8bf4dce..48a8ab9163ee 100644 --- a/x-pack/test/api_integration/apis/ml/modules/get_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/get_module.ts @@ -25,6 +25,8 @@ const moduleIds = [ 'nginx_ecs', 'sample_data_ecommerce', 'sample_data_weblogs', + 'security_linux', + 'security_windows', 'siem_auditbeat', 'siem_auditbeat_auth', 'siem_cloudtrail', diff --git a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts index d327a27bc982..64f53bbe76c5 100644 --- a/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts +++ b/x-pack/test/api_integration/apis/ml/modules/recognize_module.ts @@ -132,7 +132,7 @@ export default ({ getService }: FtrProviderContext) => { user: USER.ML_POWERUSER, expected: { responseCode: 200, - moduleIds: ['auditbeat_process_hosts_ecs', 'siem_auditbeat'], + moduleIds: ['auditbeat_process_hosts_ecs', 'security_linux', 'siem_auditbeat'], }, }, ];