[RAC][Security Solution] Adds Machine Learning rule type (#108612)

## Summary

Ports over the existing Security Solution ML Rule to the RuleRegistry.

How to test this implementation
1. Enable the following in your `kibana.dev.yml`
```
xpack.ruleRegistry.enabled: true
xpack.ruleRegistry.write.enabled: true
xpack.securitySolution.enableExperimental: ['ruleRegistryEnabled']
```

2. Create a rule by running:
```
./x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_ml.sh
```

3. Push document to anomalies index (or trigger anomaly for job id from `create_rule_ml.sh` script)


### Checklist

Delete any items that are not applicable to this PR.


- [X] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/__mocks__/rule_type.ts

@@ -10,6 +10,7 @@ import { v4 } from 'uuid';
 
 import { Logger, SavedObject } from 'kibana/server';
 import { elasticsearchServiceMock, savedObjectsClientMock } from 'src/core/server/mocks';
+import { mlPluginServerMock } from '../../../../../../ml/server/mocks';
 
 import type { IRuleDataClient } from '../../../../../../rule_registry/server';
 import { ruleRegistryMocks } from '../../../../../../rule_registry/server/mocks';
@@ -84,6 +85,7 @@ export const createRuleTypeMocks = (
       config$: mockedConfig$,
       lists: listMock.createSetup(),
       logger: loggerMock,
+      ml: mlPluginServerMock.createSetupContract(),
       ruleDataClient: ruleRegistryMocks.createRuleDataClient(
         '.alerts-security.alerts'
       ) as IRuleDataClient,

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/index.ts

@@ -7,3 +7,4 @@
 
 export { createQueryAlertType } from './query/create_query_alert_type';
 export { createIndicatorMatchAlertType } from './indicator_match/create_indicator_match_alert_type';
+export { createMlAlertType } from './ml/create_ml_alert_type';

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.test.ts

@@ -0,0 +1,112 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { mlPluginServerMock } from '../../../../../../ml/server/mocks';
+
+import { allowedExperimentalValues } from '../../../../../common/experimental_features';
+import { bulkCreateMlSignals } from '../../signals/bulk_create_ml_signals';
+
+import { createRuleTypeMocks } from '../__mocks__/rule_type';
+import { createMlAlertType } from './create_ml_alert_type';
+
+import { RuleParams } from '../../schemas/rule_schemas';
+
+jest.mock('../../signals/bulk_create_ml_signals');
+
+jest.mock('../utils/get_list_client', () => ({
+  getListClient: jest.fn().mockReturnValue({
+    listClient: {
+      getListItemIndex: jest.fn(),
+    },
+    exceptionsClient: jest.fn(),
+  }),
+}));
+
+jest.mock('../../rule_execution_log/rule_execution_log_client');
+
+jest.mock('../../signals/filters/filter_events_against_list', () => ({
+  filterEventsAgainstList: jest.fn().mockReturnValue({
+    _shards: {
+      failures: [],
+    },
+    hits: {
+      hits: [
+        {
+          is_interim: false,
+        },
+      ],
+    },
+  }),
+}));
+
+let jobsSummaryMock: jest.Mock;
+let mlMock: ReturnType<typeof mlPluginServerMock.createSetupContract>;
+
+describe('Machine Learning Alerts', () => {
+  beforeEach(() => {
+    jobsSummaryMock = jest.fn();
+    jobsSummaryMock.mockResolvedValue([
+      {
+        id: 'test-ml-job',
+        jobState: 'started',
+        datafeedState: 'started',
+      },
+    ]);
+    mlMock = mlPluginServerMock.createSetupContract();
+    mlMock.jobServiceProvider.mockReturnValue({
+      jobsSummary: jobsSummaryMock,
+    });
+
+    (bulkCreateMlSignals as jest.Mock).mockResolvedValue({
+      success: true,
+      bulkCreateDuration: 0,
+      createdItemsCount: 1,
+      createdItems: [
+        {
+          _id: '897234565234',
+          _index: 'test-index',
+          anomalyScore: 23,
+        },
+      ],
+      errors: [],
+    });
+  });
+
+  const params: Partial<RuleParams> = {
+    anomalyThreshold: 23,
+    from: 'now-45m',
+    machineLearningJobId: ['test-ml-job'],
+    to: 'now',
+    type: 'machine_learning',
+  };
+
+  it('does not send an alert when no anomalies found', async () => {
+    jobsSummaryMock.mockResolvedValue([
+      {
+        id: 'test-ml-job',
+        jobState: 'started',
+        datafeedState: 'started',
+      },
+    ]);
+    const { dependencies, executor } = createRuleTypeMocks('machine_learning', params);
+    const mlAlertType = createMlAlertType({
+      experimentalFeatures: allowedExperimentalValues,
+      lists: dependencies.lists,
+      logger: dependencies.logger,
+      mergeStrategy: 'allFields',
+      ml: mlMock,
+      ruleDataClient: dependencies.ruleDataClient,
+      ruleDataService: dependencies.ruleDataService,
+      version: '1.0.0',
+    });
+
+    dependencies.alerting.registerType(mlAlertType);
+
+    await executor({ params });
+    expect(dependencies.ruleDataClient.getWriter).not.toBeCalled();
+  });
+});

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/ml/create_ml_alert_type.ts

@@ -0,0 +1,85 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { validateNonExact } from '@kbn/securitysolution-io-ts-utils';
+import { PersistenceServices } from '../../../../../../rule_registry/server';
+import { ML_ALERT_TYPE_ID } from '../../../../../common/constants';
+import { machineLearningRuleParams, MachineLearningRuleParams } from '../../schemas/rule_schemas';
+import { mlExecutor } from '../../signals/executors/ml';
+import { createSecurityRuleTypeFactory } from '../create_security_rule_type_factory';
+import { CreateRuleOptions } from '../types';
+
+export const createMlAlertType = (createOptions: CreateRuleOptions) => {
+  const { lists, logger, mergeStrategy, ml, ruleDataClient, ruleDataService } = createOptions;
+  const createSecurityRuleType = createSecurityRuleTypeFactory({
+    lists,
+    logger,
+    mergeStrategy,
+    ruleDataClient,
+    ruleDataService,
+  });
+  return createSecurityRuleType<MachineLearningRuleParams, {}, PersistenceServices, {}>({
+    id: ML_ALERT_TYPE_ID,
+    name: 'Machine Learning Rule',
+    validate: {
+      params: {
+        validate: (object: unknown) => {
+          const [validated, errors] = validateNonExact(object, machineLearningRuleParams);
+          if (errors != null) {
+            throw new Error(errors);
+          }
+          if (validated == null) {
+            throw new Error('Validation of rule params failed');
+          }
+          return validated;
+        },
+      },
+    },
+    actionGroups: [
+      {
+        id: 'default',
+        name: 'Default',
+      },
+    ],
+    defaultActionGroupId: 'default',
+    actionVariables: {
+      context: [{ name: 'server', description: 'the server' }],
+    },
+    minimumLicenseRequired: 'basic',
+    isExportable: false,
+    producer: 'security-solution',
+    async executor(execOptions) {
+      const {
+        runOpts: {
+          buildRuleMessage,
+          bulkCreate,
+          exceptionItems,
+          listClient,
+          rule,
+          tuple,
+          wrapHits,
+        },
+        services,
+        state,
+      } = execOptions;
+
+      const result = await mlExecutor({
+        buildRuleMessage,
+        bulkCreate,
+        exceptionItems,
+        listClient,
+        logger,
+        ml,
+        rule,
+        services,
+        tuple,
+        wrapHits,
+      });
+      return { ...result, state };
+    },
+  });
+};

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/scripts/create_rule_ml.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+#
+
+curl -X POST ${KIBANA_URL}${SPACE_URL}/api/alerts/alert \
+     -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \
+     -H 'kbn-xsrf: true' \
+     -H 'Content-Type: application/json' \
+     --verbose \
+     -d '
+{
+  "params":{
+     "anomalyThreshold": 23,
+     "author": [],
+     "description": "Basic Machine Learning Rule",
+     "exceptionsList": [],
+     "falsePositives": [],
+     "from": "now-45m",
+     "immutable": false,
+     "machineLearningJobId": ["test-ml-job"],
+     "maxSignals": 101,
+     "outputIndex": "",
+     "references": [],
+     "riskScore": 23,
+     "riskScoreMapping": [],
+     "ruleId": "1781d055-5c66-4adf-9c59-fc0fa58336a5",
+     "severity": "high",
+     "severityMapping": [],
+     "threat": [],
+     "to": "now",
+     "type": "machine_learning",
+     "version": 1
+   },
+   "consumer":"alerts",
+   "alertTypeId":"siem.mlRule",
+   "schedule":{
+      "interval":"15m"
+   },
+   "actions":[],
+   "tags":[
+      "custom",
+      "ml",
+      "persistence"
+   ],
+   "notifyWhen":"onActionGroupChange",
+   "name":"Basic Machine Learning Rule"
+}'
+
+

x-pack/plugins/security_solution/server/lib/detection_engine/rule_types/types.ts

@@ -99,7 +99,7 @@ export type CreateSecurityRuleTypeFactory = (options: {
   ruleDataClient: IRuleDataClient;
   ruleDataService: IRuleDataPluginService;
 }) => <
-  TParams extends RuleParams & { index: string[] | undefined },
+  TParams extends RuleParams & { index?: string[] | undefined },
   TAlertInstanceContext extends AlertInstanceContext,
   TServices extends PersistenceServices<TAlertInstanceContext>,
   TState extends AlertTypeState
@@ -124,6 +124,7 @@ export interface CreateRuleOptions {
   lists: SetupPlugins['lists'];
   logger: Logger;
   mergeStrategy: ConfigType['alertMergeStrategy'];
+  ml?: SetupPlugins['ml'];
   ruleDataClient: IRuleDataClient;
   version: string;
   ruleDataService: IRuleDataPluginService;

x-pack/plugins/security_solution/server/plugin.ts

@@ -49,6 +49,7 @@ import { ILicense, LicensingPluginStart } from '../../licensing/server';
 import { FleetStartContract } from '../../fleet/server';
 import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server';
 import { createQueryAlertType } from './lib/detection_engine/rule_types';
+import { createMlAlertType } from './lib/detection_engine/rule_types/ml/create_ml_alert_type';
 import { initRoutes } from './routes';
 import { isAlertExecutor } from './lib/detection_engine/signals/types';
 import { signalRulesAlertType } from './lib/detection_engine/signals/signal_rule_alert_type';
@@ -65,6 +66,7 @@ import {
   QUERY_ALERT_TYPE_ID,
   DEFAULT_SPACE_ID,
   INDICATOR_ALERT_TYPE_ID,
+  ML_ALERT_TYPE_ID,
 } from '../common/constants';
 import { registerEndpointRoutes } from './endpoint/routes/metadata';
 import { registerLimitedConcurrencyRoutes } from './endpoint/routes/limited_concurrency';
@@ -129,6 +131,7 @@ export interface PluginSetup {}
 
 // eslint-disable-next-line @typescript-eslint/no-empty-interface
 export interface PluginStart {}
+
 export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, StartPlugins> {
   private readonly logger: Logger;
   private readonly config: ConfigType;
@@ -246,6 +249,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
         lists: plugins.lists,
         logger: this.logger,
         mergeStrategy: this.config.alertMergeStrategy,
+        ml: plugins.ml,
         ruleDataClient,
         ruleDataService,
         version: this.context.env.packageInfo.version,
@@ -253,6 +257,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
 
       this.setupPlugins.alerting.registerType(createQueryAlertType(createRuleOptions));
       this.setupPlugins.alerting.registerType(createIndicatorMatchAlertType(createRuleOptions));
+      this.setupPlugins.alerting.registerType(createMlAlertType(createRuleOptions));
     }
 
     // TODO We need to get the endpoint routes inside of initRoutes
@@ -272,7 +277,7 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     registerTrustedAppsRoutes(router, endpointContext);
     registerActionRoutes(router, endpointContext);
 
-    const racRuleTypes = [QUERY_ALERT_TYPE_ID, INDICATOR_ALERT_TYPE_ID];
+    const racRuleTypes = [QUERY_ALERT_TYPE_ID, INDICATOR_ALERT_TYPE_ID, ML_ALERT_TYPE_ID];
     const ruleTypes = [
       SIGNALS_ID,
       NOTIFICATIONS_ID,