[Security Solution][Detections] Adoption telemetry (#71102)

* style: sort plugin interface

* WIP: UsageCollector for Security Adoption

This uses ML and raw ES calls to query our ML Jobs and Rules, and parse
them into a format to be consumed by telemetry.

Still to come:
* initialization
* tests

* Initialize usage collectors during plugin setup

* Rename usage key

The service seems to convert colons to underscores, so let's just use an
underscure.

* Collector is ready if we have a kibana index

* Refactor collector to generate options in a function

This allows us to test our adherence to the collector API, focusing
particularly on the fetch function.

* Refactor usage collector in anticipation of endpoint data

We're going to have our usage data under one key corresponding to the
app, so this nests the existing data under a 'detections' key while
allowing another fetching function to be plugged into the
main collector under a separate key.

* Update our collector to satisfy telemetry tooling

* inlines collector options
* inlines schema object
* makes DetectionsUsage an interface instead of a type alias

* Extracts telemetry mappings via scripts/telemetry_extract

* Refactor detections usage logic to perform one loop instead of two

We were previously performing two loops over each set of data: one to
format it down to just the data we need, and another to convert that
into usage data. We now perform both steps within a single loop.

* Refactor detections telemetry to be nested

* Extract new nested detections telemetry mappings

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/security_solution/server/plugin.ts

@@ -16,6 +16,7 @@ import {
   PluginInitializerContext,
   SavedObjectsClient,
 } from '../../../../src/core/server';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { PluginSetupContract as AlertingSetup } from '../../alerts/server';
 import { SecurityPluginSetup as SecuritySetup } from '../../security/server';
 import { PluginSetupContract as FeaturesSetup } from '../../features/server';
@@ -46,17 +47,19 @@ import { ArtifactClient, ManifestManager } from './endpoint/services';
 import { EndpointAppContextService } from './endpoint/endpoint_app_context_services';
 import { EndpointAppContext } from './endpoint/types';
 import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts';
+import { initUsageCollectors } from './usage';
 
 export interface SetupPlugins {
   alerts: AlertingSetup;
   encryptedSavedObjects?: EncryptedSavedObjectsSetup;
   features: FeaturesSetup;
   licensing: LicensingPluginSetup;
+  lists?: ListPluginSetup;
+  ml?: MlSetup;
   security?: SecuritySetup;
   spaces?: SpacesSetup;
   taskManager?: TaskManagerSetupContract;
-  ml?: MlSetup;
-  lists?: ListPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export interface StartPlugins {
@@ -106,9 +109,15 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     this.logger.debug('plugin setup');
 
     const config = await this.config$.pipe(first()).toPromise();
+    const globalConfig = await this.context.config.legacy.globalConfig$.pipe(first()).toPromise();
 
     initSavedObjects(core.savedObjects);
     initUiSettings(core.uiSettings);
+    initUsageCollectors({
+      kibanaIndex: globalConfig.kibana.index,
+      ml: plugins.ml,
+      usageCollection: plugins.usageCollection,
+    });
 
     const endpointContext: EndpointAppContext = {
       logFactory: this.context.logger,

x-pack/plugins/security_solution/server/usage/collector.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from '../../../../../src/core/server';
+import { CollectorDependencies } from './types';
+import { DetectionsUsage, fetchDetectionsUsage } from './detections';
+
+export type RegisterCollector = (deps: CollectorDependencies) => void;
+export interface UsageData {
+  detections: DetectionsUsage;
+}
+
+export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => {
+  if (!usageCollection) {
+    return;
+  }
+
+  const collector = usageCollection.makeUsageCollector<UsageData>({
+    type: 'security_solution',
+    schema: {
+      detections: {
+        detection_rules: {
+          custom: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+          elastic: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+        },
+        ml_jobs: {
+          custom: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+          elastic: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+        },
+      },
+    },
+    isReady: () => kibanaIndex.length > 0,
+    fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => ({
+      detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
+    }),
+  });
+
+  usageCollection.registerCollector(collector);
+};

x-pack/plugins/security_solution/server/usage/detections.mocks.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
+
+export const getMockJobSummaryResponse = () => [
+  {
+    id: 'linux_anomalous_network_activity_ecs',
+    description:
+      'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)',
+    groups: ['auditbeat', 'process', 'siem'],
+    processed_record_count: 141889,
+    memory_status: 'ok',
+    jobState: 'opened',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-linux_anomalous_network_activity_ecs',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'started',
+    latestTimestampMs: 1594085401911,
+    earliestTimestampMs: 1593054845656,
+    latestResultsTimestampMs: 1594085401911,
+    isSingleMetricViewerJob: true,
+    nodeName: 'node',
+  },
+  {
+    id: 'linux_anomalous_network_port_activity_ecs',
+    description:
+      'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)',
+    groups: ['auditbeat', 'process', 'siem'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'closed',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'stopped',
+    isSingleMetricViewerJob: true,
+  },
+  {
+    id: 'other_job',
+    description: 'a job that is custom',
+    groups: ['auditbeat', 'process'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'closed',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-other',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'stopped',
+    isSingleMetricViewerJob: true,
+  },
+  {
+    id: 'another_job',
+    description: 'another job that is custom',
+    groups: ['auditbeat', 'process'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'opened',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-another',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'started',
+    isSingleMetricViewerJob: true,
+  },
+];
+
+export const getMockListModulesResponse = () => [
+  {
+    id: 'siem_auditbeat',
+    title: 'SIEM Auditbeat',
+    description:
+      'Detect suspicious network activity and unusual processes in Auditbeat data (beta).',
+    type: 'Auditbeat data',
+    logoFile: 'logo.json',
+    defaultIndexPattern: 'auditbeat-*',
+    query: {
+      bool: {
+        filter: [
+          {
+            term: {
+              'agent.type': 'auditbeat',
+            },
+          },
+        ],
+      },
+    },
+    jobs: [
+      {
+        id: 'linux_anomalous_network_activity_ecs',
+        config: {
+          job_type: 'anomaly_detector',
+          description:
+            'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)',
+          groups: ['siem', 'auditbeat', 'process'],
+          analysis_config: {
+            bucket_span: '15m',
+            detectors: [
+              {
+                detector_description: 'rare by "process.name"',
+                function: 'rare',
+                by_field_name: 'process.name',
+              },
+            ],
+            influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'],
+          },
+          allow_lazy_open: true,
+          analysis_limits: {
+            model_memory_limit: '64mb',
+          },
+          data_description: {
+            time_field: '@timestamp',
+          },
+        },
+      },
+      {
+        id: 'linux_anomalous_network_port_activity_ecs',
+        config: {
+          job_type: 'anomaly_detector',
+          description:
+            'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)',
+          groups: ['siem', 'auditbeat', 'network'],
+          analysis_config: {
+            bucket_span: '15m',
+            detectors: [
+              {
+                detector_description: 'rare by "destination.port"',
+                function: 'rare',
+                by_field_name: 'destination.port',
+              },
+            ],
+            influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'],
+          },
+          allow_lazy_open: true,
+          analysis_limits: {
+            model_memory_limit: '32mb',
+          },
+          data_description: {
+            time_field: '@timestamp',
+          },
+        },
+      },
+    ],
+    datafeeds: [],
+    kibana: {},
+  },
+];
+
+export const getMockRulesResponse = () => ({
+  hits: {
+    hits: [
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+    ],
+  },
+});

x-pack/plugins/security_solution/server/usage/detections.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from '../../../../../src/core/server';
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+import { jobServiceProvider } from '../../../ml/server/models/job_service';
+import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
+import { mlServicesMock } from '../lib/machine_learning/mocks';
+import {
+  getMockJobSummaryResponse,
+  getMockListModulesResponse,
+  getMockRulesResponse,
+} from './detections.mocks';
+import { fetchDetectionsUsage } from './detections';
+
+jest.mock('../../../ml/server/models/job_service');
+jest.mock('../../../ml/server/models/data_recognizer');
+
+describe('Detections Usage', () => {
+  describe('fetchDetectionsUsage()', () => {
+    let callClusterMock: jest.Mocked<LegacyAPICaller>;
+    let mlMock: ReturnType<typeof mlServicesMock.create>;
+
+    beforeEach(() => {
+      callClusterMock = elasticsearchServiceMock.createLegacyClusterClient().callAsInternalUser;
+      mlMock = mlServicesMock.create();
+    });
+
+    it('returns zeroed counts if both calls are empty', async () => {
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual({
+        detection_rules: {
+          custom: {
+            enabled: 0,
+            disabled: 0,
+          },
+          elastic: {
+            enabled: 0,
+            disabled: 0,
+          },
+        },
+        ml_jobs: {
+          custom: {
+            enabled: 0,
+            disabled: 0,
+          },
+          elastic: {
+            enabled: 0,
+            disabled: 0,
+          },
+        },
+      });
+    });
+
+    it('tallies rules data given rules results', async () => {
+      (callClusterMock as jest.Mock).mockResolvedValue(getMockRulesResponse());
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          detection_rules: {
+            custom: {
+              enabled: 1,
+              disabled: 1,
+            },
+            elastic: {
+              enabled: 2,
+              disabled: 3,
+            },
+          },
+        })
+      );
+    });
+
+    it('tallies jobs data given jobs results', async () => {
+      const mockJobSummary = jest.fn().mockResolvedValue(getMockJobSummaryResponse());
+      const mockListModules = jest.fn().mockResolvedValue(getMockListModulesResponse());
+      (jobServiceProvider as jest.Mock).mockImplementation(() => ({
+        jobsSummary: mockJobSummary,
+      }));
+      (DataRecognizer as jest.Mock).mockImplementation(() => ({
+        listModules: mockListModules,
+      }));
+
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          ml_jobs: {
+            custom: {
+              enabled: 1,
+              disabled: 1,
+            },
+            elastic: {
+              enabled: 1,
+              disabled: 1,
+            },
+          },
+        })
+      );
+    });
+  });
+});

x-pack/plugins/security_solution/server/usage/detections.ts

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from '../../../../../src/core/server';
+import { getMlJobsUsage, getRulesUsage } from './detections_helpers';
+import { MlPluginSetup } from '../../../ml/server';
+
+interface FeatureUsage {
+  enabled: number;
+  disabled: number;
+}
+
+export interface DetectionRulesUsage {
+  custom: FeatureUsage;
+  elastic: FeatureUsage;
+}
+
+export interface MlJobsUsage {
+  custom: FeatureUsage;
+  elastic: FeatureUsage;
+}
+
+export interface DetectionsUsage {
+  detection_rules: DetectionRulesUsage;
+  ml_jobs: MlJobsUsage;
+}
+
+export const fetchDetectionsUsage = async (
+  kibanaIndex: string,
+  callCluster: LegacyAPICaller,
+  ml: MlPluginSetup | undefined
+): Promise<DetectionsUsage> => {
+  const rulesUsage = await getRulesUsage(kibanaIndex, callCluster);
+  const mlJobsUsage = await getMlJobsUsage(ml);
+  return { detection_rules: rulesUsage, ml_jobs: mlJobsUsage };
+};

x-pack/plugins/security_solution/server/usage/detections_helpers.ts

@@ -0,0 +1,188 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SearchParams } from 'elasticsearch';
+
+import { LegacyAPICaller, SavedObjectsClient } from '../../../../../src/core/server';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { jobServiceProvider } from '../../../ml/server/models/job_service';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
+import { MlPluginSetup } from '../../../ml/server';
+import { SIGNALS_ID, INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
+import { DetectionRulesUsage, MlJobsUsage } from './detections';
+import { isJobStarted } from '../../common/machine_learning/helpers';
+
+interface DetectionsMetric {
+  isElastic: boolean;
+  isEnabled: boolean;
+}
+
+const isElasticRule = (tags: string[]) => tags.includes(`${INTERNAL_IMMUTABLE_KEY}:true`);
+
+const initialRulesUsage: DetectionRulesUsage = {
+  custom: {
+    enabled: 0,
+    disabled: 0,
+  },
+  elastic: {
+    enabled: 0,
+    disabled: 0,
+  },
+};
+
+const initialMlJobsUsage: MlJobsUsage = {
+  custom: {
+    enabled: 0,
+    disabled: 0,
+  },
+  elastic: {
+    enabled: 0,
+    disabled: 0,
+  },
+};
+
+const updateRulesUsage = (
+  ruleMetric: DetectionsMetric,
+  usage: DetectionRulesUsage
+): DetectionRulesUsage => {
+  const { isEnabled, isElastic } = ruleMetric;
+  if (isEnabled && isElastic) {
+    return {
+      ...usage,
+      elastic: {
+        ...usage.elastic,
+        enabled: usage.elastic.enabled + 1,
+      },
+    };
+  } else if (!isEnabled && isElastic) {
+    return {
+      ...usage,
+      elastic: {
+        ...usage.elastic,
+        disabled: usage.elastic.disabled + 1,
+      },
+    };
+  } else if (isEnabled && !isElastic) {
+    return {
+      ...usage,
+      custom: {
+        ...usage.custom,
+        enabled: usage.custom.enabled + 1,
+      },
+    };
+  } else if (!isEnabled && !isElastic) {
+    return {
+      ...usage,
+      custom: {
+        ...usage.custom,
+        disabled: usage.custom.disabled + 1,
+      },
+    };
+  } else {
+    return usage;
+  }
+};
+
+const updateMlJobsUsage = (jobMetric: DetectionsMetric, usage: MlJobsUsage): MlJobsUsage => {
+  const { isEnabled, isElastic } = jobMetric;
+  if (isEnabled && isElastic) {
+    return {
+      ...usage,
+      elastic: {
+        ...usage.elastic,
+        enabled: usage.elastic.enabled + 1,
+      },
+    };
+  } else if (!isEnabled && isElastic) {
+    return {
+      ...usage,
+      elastic: {
+        ...usage.elastic,
+        disabled: usage.elastic.disabled + 1,
+      },
+    };
+  } else if (isEnabled && !isElastic) {
+    return {
+      ...usage,
+      custom: {
+        ...usage.custom,
+        enabled: usage.custom.enabled + 1,
+      },
+    };
+  } else if (!isEnabled && !isElastic) {
+    return {
+      ...usage,
+      custom: {
+        ...usage.custom,
+        disabled: usage.custom.disabled + 1,
+      },
+    };
+  } else {
+    return usage;
+  }
+};
+
+export const getRulesUsage = async (
+  index: string,
+  callCluster: LegacyAPICaller
+): Promise<DetectionRulesUsage> => {
+  let rulesUsage: DetectionRulesUsage = initialRulesUsage;
+  const ruleSearchOptions: SearchParams = {
+    body: { query: { bool: { filter: { term: { 'alert.alertTypeId': SIGNALS_ID } } } } },
+    filterPath: ['hits.hits._source.alert.enabled', 'hits.hits._source.alert.tags'],
+    ignoreUnavailable: true,
+    index,
+    size: 10000, // elasticsearch index.max_result_window default value
+  };
+
+  try {
+    const ruleResults = await callCluster<{ alert: { enabled: boolean; tags: string[] } }>(
+      'search',
+      ruleSearchOptions
+    );
+
+    if (ruleResults.hits?.hits?.length > 0) {
+      rulesUsage = ruleResults.hits.hits.reduce((usage, hit) => {
+        const isElastic = isElasticRule(hit._source.alert.tags);
+        const isEnabled = hit._source.alert.enabled;
+
+        return updateRulesUsage({ isElastic, isEnabled }, usage);
+      }, initialRulesUsage);
+    }
+  } catch (e) {
+    // ignore failure, usage will be zeroed
+  }
+
+  return rulesUsage;
+};
+
+export const getMlJobsUsage = async (ml: MlPluginSetup | undefined): Promise<MlJobsUsage> => {
+  let jobsUsage: MlJobsUsage = initialMlJobsUsage;
+
+  if (ml) {
+    try {
+      const mlCaller = ml.mlClient.callAsInternalUser;
+      const modules = await new DataRecognizer(
+        mlCaller,
+        ({} as unknown) as SavedObjectsClient
+      ).listModules();
+      const moduleJobs = modules.flatMap((module) => module.jobs);
+      const jobs = await jobServiceProvider(mlCaller).jobsSummary(['siem']);
+
+      jobsUsage = jobs.reduce((usage, job) => {
+        const isElastic = moduleJobs.some((moduleJob) => moduleJob.id === job.id);
+        const isEnabled = isJobStarted(job.jobState, job.datafeedState);
+
+        return updateMlJobsUsage({ isElastic, isEnabled }, usage);
+      }, initialMlJobsUsage);
+    } catch (e) {
+      // ignore failure, usage will be zeroed
+    }
+  }
+
+  return jobsUsage;
+};

x-pack/plugins/security_solution/server/usage/index.ts

@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { CollectorDependencies } from './types';
+import { registerCollector } from './collector';
+
+export type InitUsageCollectors = (deps: CollectorDependencies) => void;
+
+export const initUsageCollectors: InitUsageCollectors = (dependencies) => {
+  registerCollector(dependencies);
+};

x-pack/plugins/security_solution/server/usage/types.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SetupPlugins } from '../plugin';
+
+export type CollectorDependencies = { kibanaIndex: string } & Pick<
+  SetupPlugins,
+  'ml' | 'usageCollection'
+>;

x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json

@@ -164,6 +164,62 @@
         }
       }
     },
+    "security_solution": {
+      "properties": {
+        "detections": {
+          "properties": {
+            "detection_rules": {
+              "properties": {
+                "custom": {
+                  "properties": {
+                    "enabled": {
+                      "type": "long"
+                    },
+                    "disabled": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "elastic": {
+                  "properties": {
+                    "enabled": {
+                      "type": "long"
+                    },
+                    "disabled": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            },
+            "ml_jobs": {
+              "properties": {
+                "custom": {
+                  "properties": {
+                    "enabled": {
+                      "type": "long"
+                    },
+                    "disabled": {
+                      "type": "long"
+                    }
+                  }
+                },
+                "elastic": {
+                  "properties": {
+                    "enabled": {
+                      "type": "long"
+                    },
+                    "disabled": {
+                      "type": "long"
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    },
     "spaces": {
       "properties": {
         "usesFeatureControls": {