Ability to get scoped call cluster from alerting and action executors (#64432)

* Initial work

* Rename to getScopedCallCluster

* Fix typecheck

* Fix more type check issues

* Add tests

* Add docs

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/actions/README.md

@@ -143,7 +143,8 @@ This is the primary function for an action type. Whenever the action needs to ex
 | actionId                                | The action saved object id that the action type is executing for.                                                                                                                                                                                                                                                                               |
 | config                                  | The decrypted configuration given to an action. This comes from the action saved object that is partially or fully encrypted within the data store. If you would like to validate the config before being passed to the executor, define `validate.config` within the action type.                                                              |
 | params                                  | Parameters for the execution. These will be given at execution time by either an alert or manually provided when calling the plugin provided execute function.                                                                                                                                                                                  |
-| services.callCluster(path, opts)        | Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana.<br><br>**NOTE**: This currently authenticates as the Kibana internal user, but will change in a future PR.                                                                                              |
+| services.callCluster(path, opts)        | Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana but runs in the context of the user who is calling the action when security is enabled.|
+| services.getScopedCallCluster           | This function scopes an instance of CallCluster by returning a `callCluster(path, opts)` function that runs in the context of the user who is calling the action when security is enabled. This must only be called with instances of CallCluster provided by core.|
 | services.savedObjectsClient             | This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>The scope of the saved objects client is tied to the user in context calling the execute API or the API key provided to the execute plugin function (only when security isenabled). |
 | services.log(tags, [data], [timestamp]) | Use this to create server logs. (This is the same function as server.log)                                                                                                                                                                                                                                                                       |
 

x-pack/plugins/actions/server/builtin_action_types/email.test.ts

@@ -9,13 +9,13 @@ jest.mock('./lib/send_email', () => ({
 }));
 
 import { Logger } from '../../../../../src/core/server';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 
 import { ActionType, ActionTypeExecutorOptions } from '../types';
 import { actionsConfigMock } from '../actions_config.mock';
 import { validateConfig, validateSecrets, validateParams } from '../lib';
 import { createActionTypeRegistry } from './index.test';
 import { sendEmail } from './lib/send_email';
+import { actionsMock } from '../mocks';
 import {
   ActionParamsType,
   ActionTypeConfigType,
@@ -26,13 +26,8 @@ import {
 const sendEmailMock = sendEmail as jest.Mock;
 
 const ACTION_TYPE_ID = '.email';
-const NO_OP_FN = () => {};
 
-const services = {
-  log: NO_OP_FN,
-  callCluster: async (path: string, opts: unknown) => {},
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services = actionsMock.createServices();
 
 let actionType: ActionType;
 let mockedLogger: jest.Mocked<Logger>;

x-pack/plugins/actions/server/builtin_action_types/es_index.test.ts

@@ -10,18 +10,13 @@ jest.mock('./lib/send_email', () => ({
 
 import { ActionType, ActionTypeExecutorOptions } from '../types';
 import { validateConfig, validateParams } from '../lib';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 import { createActionTypeRegistry } from './index.test';
 import { ActionParamsType, ActionTypeConfigType } from './es_index';
+import { actionsMock } from '../mocks';
 
 const ACTION_TYPE_ID = '.index';
-const NO_OP_FN = () => {};
 
-const services = {
-  log: NO_OP_FN,
-  callCluster: jest.fn(),
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services = actionsMock.createServices();
 
 let actionType: ActionType;
 
@@ -196,9 +191,9 @@ describe('execute()', () => {
     await actionType.executor(executorOptions);
 
     const calls = services.callCluster.mock.calls;
-    const timeValue = calls[0][1].body[1].field_to_use_for_time;
+    const timeValue = calls[0][1]?.body[1].field_to_use_for_time;
     expect(timeValue).toBeInstanceOf(Date);
-    delete calls[0][1].body[1].field_to_use_for_time;
+    delete calls[0][1]?.body[1].field_to_use_for_time;
     expect(calls).toMatchInlineSnapshot(`
         Array [
           Array [

x-pack/plugins/actions/server/builtin_action_types/es_index.ts

@@ -72,7 +72,7 @@ async function executor(
     bulkBody.push(document);
   }
 
-  const bulkParams: unknown = {
+  const bulkParams = {
     index,
     body: bulkBody,
     refresh: config.refresh,

x-pack/plugins/actions/server/builtin_action_types/pagerduty.test.ts

@@ -11,20 +11,17 @@ jest.mock('./lib/post_pagerduty', () => ({
 import { getActionType } from './pagerduty';
 import { ActionType, Services, ActionTypeExecutorOptions } from '../types';
 import { validateConfig, validateSecrets, validateParams } from '../lib';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 import { postPagerduty } from './lib/post_pagerduty';
 import { createActionTypeRegistry } from './index.test';
 import { Logger } from '../../../../../src/core/server';
 import { actionsConfigMock } from '../actions_config.mock';
+import { actionsMock } from '../mocks';
 
 const postPagerdutyMock = postPagerduty as jest.Mock;
 
 const ACTION_TYPE_ID = '.pagerduty';
 
-const services: Services = {
-  callCluster: async (path: string, opts: unknown) => {},
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services: Services = actionsMock.createServices();
 
 let actionType: ActionType;
 let mockedLogger: jest.Mocked<Logger>;

x-pack/plugins/actions/server/builtin_action_types/server_log.test.ts

@@ -7,8 +7,8 @@
 import { ActionType } from '../types';
 import { validateParams } from '../lib';
 import { Logger } from '../../../../../src/core/server';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 import { createActionTypeRegistry } from './index.test';
+import { actionsMock } from '../mocks';
 
 const ACTION_TYPE_ID = '.server-log';
 
@@ -90,10 +90,7 @@ describe('execute()', () => {
     const actionId = 'some-id';
     await actionType.executor({
       actionId,
-      services: {
-        callCluster: async (path: string, opts: unknown) => {},
-        savedObjectsClient: savedObjectsClientMock.create(),
-      },
+      services: actionsMock.createServices(),
       params: { message: 'message text here', level: 'info' },
       config: {},
       secrets: {},

x-pack/plugins/actions/server/builtin_action_types/servicenow/index.test.ts

@@ -7,9 +7,9 @@
 import { getActionType } from '.';
 import { ActionType, Services, ActionTypeExecutorOptions } from '../../types';
 import { validateConfig, validateSecrets, validateParams } from '../../lib';
-import { savedObjectsClientMock } from '../../../../../../src/core/server/mocks';
 import { createActionTypeRegistry } from '../index.test';
 import { actionsConfigMock } from '../../actions_config.mock';
+import { actionsMock } from '../../mocks';
 
 import { ACTION_TYPE_ID } from './constants';
 import * as i18n from './translations';
@@ -21,10 +21,7 @@ jest.mock('./action_handlers');
 
 const handleIncidentMock = handleIncident as jest.Mock;
 
-const services: Services = {
-  callCluster: async (path: string, opts: unknown) => {},
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services: Services = actionsMock.createServices();
 
 let actionType: ActionType;
 

x-pack/plugins/actions/server/builtin_action_types/slack.test.ts

@@ -10,17 +10,14 @@ import {
   ActionTypeExecutorOptions,
   ActionTypeExecutorResult,
 } from '../types';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 import { validateParams, validateSecrets } from '../lib';
 import { getActionType } from './slack';
 import { actionsConfigMock } from '../actions_config.mock';
+import { actionsMock } from '../mocks';
 
 const ACTION_TYPE_ID = '.slack';
 
-const services: Services = {
-  callCluster: async (path: string, opts: unknown) => {},
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services: Services = actionsMock.createServices();
 
 let actionType: ActionType;
 

x-pack/plugins/actions/server/builtin_action_types/webhook.test.ts

@@ -11,20 +11,17 @@ jest.mock('axios', () => ({
 import { getActionType } from './webhook';
 import { ActionType, Services } from '../types';
 import { validateConfig, validateSecrets, validateParams } from '../lib';
-import { savedObjectsClientMock } from '../../../../../src/core/server/mocks';
 import { actionsConfigMock } from '../actions_config.mock';
 import { createActionTypeRegistry } from './index.test';
 import { Logger } from '../../../../../src/core/server';
+import { actionsMock } from '../mocks';
 import axios from 'axios';
 
 const axiosRequestMock = axios.request as jest.Mock;
 
 const ACTION_TYPE_ID = '.webhook';
 
-const services: Services = {
-  callCluster: async (path: string, opts: unknown) => {},
-  savedObjectsClient: savedObjectsClientMock.create(),
-};
+const services: Services = actionsMock.createServices();
 
 let actionType: ActionType;
 let mockedLogger: jest.Mocked<Logger>;

x-pack/plugins/actions/server/lib/action_executor.test.ts

@@ -9,21 +9,15 @@ import { schema } from '@kbn/config-schema';
 import { ActionExecutor } from './action_executor';
 import { actionTypeRegistryMock } from '../action_type_registry.mock';
 import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks';
-import { savedObjectsClientMock, loggingServiceMock } from '../../../../../src/core/server/mocks';
+import { loggingServiceMock } from '../../../../../src/core/server/mocks';
 import { eventLoggerMock } from '../../../event_log/server/mocks';
 import { spacesServiceMock } from '../../../spaces/server/spaces_service/spaces_service.mock';
 import { ActionType } from '../types';
+import { actionsMock } from '../mocks';
 
 const actionExecutor = new ActionExecutor({ isESOUsingEphemeralEncryptionKey: false });
-const savedObjectsClient = savedObjectsClientMock.create();
-
-function getServices() {
-  return {
-    savedObjectsClient,
-    log: jest.fn(),
-    callCluster: jest.fn(),
-  };
-}
+const services = actionsMock.createServices();
+const savedObjectsClient = services.savedObjectsClient;
 const encryptedSavedObjectsPlugin = encryptedSavedObjectsMock.createStart();
 const actionTypeRegistry = actionTypeRegistryMock.create();
 
@@ -39,7 +33,7 @@ const spacesMock = spacesServiceMock.createSetupContract();
 actionExecutor.initialize({
   logger: loggingServiceMock.create().get(),
   spaces: spacesMock,
-  getServices,
+  getServices: () => services,
   actionTypeRegistry,
   encryptedSavedObjectsPlugin,
   eventLogger: eventLoggerMock.create(),
@@ -273,7 +267,7 @@ test('throws an error when passing isESOUsingEphemeralEncryptionKey with value o
   customActionExecutor.initialize({
     logger: loggingServiceMock.create().get(),
     spaces: spacesMock,
-    getServices,
+    getServices: () => services,
     actionTypeRegistry,
     encryptedSavedObjectsPlugin,
     eventLogger: eventLoggerMock.create(),

x-pack/plugins/actions/server/mocks.ts

@@ -6,6 +6,11 @@
 
 import { actionsClientMock } from './actions_client.mock';
 import { PluginSetupContract, PluginStartContract } from './plugin';
+import { Services } from './types';
+import {
+  elasticsearchServiceMock,
+  savedObjectsClientMock,
+} from '../../../../src/core/server/mocks';
 
 export { actionsClientMock };
 
@@ -27,7 +32,19 @@ const createStartMock = () => {
   return mock;
 };
 
+const createServicesMock = () => {
+  const mock: jest.Mocked<Services & {
+    savedObjectsClient: ReturnType<typeof savedObjectsClientMock.create>;
+  }> = {
+    callCluster: elasticsearchServiceMock.createScopedClusterClient().callAsCurrentUser,
+    getScopedCallCluster: jest.fn(),
+    savedObjectsClient: savedObjectsClientMock.create(),
+  };
+  return mock;
+};
+
 export const actionsMock = {
+  createServices: createServicesMock,
   createSetup: createSetupMock,
   createStart: createStartMock,
 };

x-pack/plugins/actions/server/plugin.ts

@@ -18,6 +18,7 @@ import {
   IContextProvider,
   SavedObjectsServiceStart,
   ElasticsearchServiceStart,
+  IClusterClient,
 } from '../../../../src/core/server';
 
 import {
@@ -302,6 +303,9 @@ export class ActionsPlugin implements Plugin<Promise<PluginSetupContract>, Plugi
     return request => ({
       callCluster: elasticsearch.legacy.client.asScoped(request).callAsCurrentUser,
       savedObjectsClient: savedObjects.getScopedClient(request),
+      getScopedCallCluster(clusterClient: IClusterClient) {
+        return clusterClient.asScoped(request).callAsCurrentUser;
+      },
     });
   }
 

x-pack/plugins/actions/server/types.ts

@@ -4,15 +4,17 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import {
-  SavedObjectsClientContract,
-  SavedObjectAttributes,
-  KibanaRequest,
-} from '../../../../src/core/server';
 import { ActionTypeRegistry } from './action_type_registry';
 import { PluginSetupContract, PluginStartContract } from './plugin';
 import { ActionsClient } from './actions_client';
 import { LicenseType } from '../../licensing/common/types';
+import {
+  IClusterClient,
+  IScopedClusterClient,
+  KibanaRequest,
+  SavedObjectsClientContract,
+  SavedObjectAttributes,
+} from '../../../../src/core/server';
 
 export type WithoutQueryAndParams<T> = Pick<T, Exclude<keyof T, 'query' | 'params'>>;
 export type GetServicesFunction = (request: KibanaRequest) => Services;
@@ -21,8 +23,9 @@ export type GetBasePathFunction = (spaceId?: string) => string;
 export type SpaceIdToNamespaceFunction = (spaceId?: string) => string | undefined;
 
 export interface Services {
-  callCluster(path: string, opts: unknown): Promise<unknown>;
+  callCluster: IScopedClusterClient['callAsCurrentUser'];
   savedObjectsClient: SavedObjectsClientContract;
+  getScopedCallCluster(clusterClient: IClusterClient): IScopedClusterClient['callAsCurrentUser'];
 }
 
 declare module 'src/core/server' {

x-pack/plugins/alerting/README.md

@@ -101,6 +101,7 @@ This is the primary function for an alert type. Whenever the alert needs to exec
 |---|---|
 |services.callCluster(path, opts)|Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana but in the context of the user who created the alert when security is enabled.|
 |services.savedObjectsClient|This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>The scope of the saved objects client is tied to the user who created the alert (only when security isenabled).|
+|services.getScopedCallCluster|This function scopes an instance of CallCluster by returning a `callCluster(path, opts)` function that runs in the context of the user who created the alert when security is enabled. This must only be called with instances of CallCluster provided by core.|
 |services.log(tags, [data], [timestamp])|Use this to create server logs. (This is the same function as server.log)|
 |startedAt|The date and time the alert type started execution.|
 |previousStartedAt|The previous date and time the alert type started a successful execution.|

x-pack/plugins/alerting/server/mocks.ts

@@ -6,8 +6,11 @@
 
 import { alertsClientMock } from './alerts_client.mock';
 import { PluginSetupContract, PluginStartContract } from './plugin';
-import { savedObjectsClientMock } from '../../../../src/core/server/mocks';
 import { AlertInstance } from './alert_instance';
+import {
+  elasticsearchServiceMock,
+  savedObjectsClientMock,
+} from '../../../../src/core/server/mocks';
 
 export { alertsClientMock };
 
@@ -55,7 +58,8 @@ const createAlertServicesMock = () => {
     alertInstanceFactory: jest
       .fn<jest.Mocked<AlertInstance>, [string]>()
       .mockReturnValue(alertInstanceFactoryMock),
-    callCluster: jest.fn(),
+    callCluster: elasticsearchServiceMock.createScopedClusterClient().callAsCurrentUser,
+    getScopedCallCluster: jest.fn(),
     savedObjectsClient: savedObjectsClientMock.create(),
   };
 };

x-pack/plugins/alerting/server/plugin.ts

@@ -29,6 +29,7 @@ import {
   RequestHandler,
   SharedGlobalConfig,
   ElasticsearchServiceStart,
+  IClusterClient,
 } from '../../../../src/core/server';
 
 import {
@@ -270,6 +271,9 @@ export class AlertingPlugin {
     return request => ({
       callCluster: elasticsearch.legacy.client.asScoped(request).callAsCurrentUser,
       savedObjectsClient: savedObjects.getScopedClient(request),
+      getScopedCallCluster(clusterClient: IClusterClient) {
+        return clusterClient.asScoped(request).callAsCurrentUser;
+      },
     });
   }
 

x-pack/plugins/alerting/server/task_runner/task_runner.test.ts

@@ -11,9 +11,10 @@ import { ConcreteTaskInstance, TaskStatus } from '../../../../plugins/task_manag
 import { TaskRunnerContext } from './task_runner_factory';
 import { TaskRunner } from './task_runner';
 import { encryptedSavedObjectsMock } from '../../../../plugins/encrypted_saved_objects/server/mocks';
-import { savedObjectsClientMock, loggingServiceMock } from '../../../../../src/core/server/mocks';
+import { loggingServiceMock } from '../../../../../src/core/server/mocks';
 import { PluginStartContract as ActionsPluginStart } from '../../../actions/server';
 import { actionsMock } from '../../../actions/server/mocks';
+import { alertsMock } from '../mocks';
 import { eventLoggerMock } from '../../../event_log/server/event_logger.mock';
 import { IEventLogger } from '../../../event_log/server';
 import { SavedObjectsErrorHelpers } from '../../../../../src/core/server';
@@ -52,13 +53,9 @@ describe('Task Runner', () => {
 
   afterAll(() => fakeTimer.restore());
 
-  const savedObjectsClient = savedObjectsClientMock.create();
   const encryptedSavedObjectsPlugin = encryptedSavedObjectsMock.createStart();
-  const services = {
-    log: jest.fn(),
-    callCluster: jest.fn(),
-    savedObjectsClient,
-  };
+  const services = alertsMock.createAlertServices();
+  const savedObjectsClient = services.savedObjectsClient;
 
   const taskRunnerFactoryInitializerParams: jest.Mocked<TaskRunnerContext> & {
     actionsPlugin: jest.Mocked<ActionsPluginStart>;

x-pack/plugins/alerting/server/task_runner/task_runner_factory.test.ts

@@ -8,8 +8,9 @@ import sinon from 'sinon';
 import { ConcreteTaskInstance, TaskStatus } from '../../../../plugins/task_manager/server';
 import { TaskRunnerContext, TaskRunnerFactory } from './task_runner_factory';
 import { encryptedSavedObjectsMock } from '../../../../plugins/encrypted_saved_objects/server/mocks';
-import { savedObjectsClientMock, loggingServiceMock } from '../../../../../src/core/server/mocks';
+import { loggingServiceMock } from '../../../../../src/core/server/mocks';
 import { actionsMock } from '../../../actions/server/mocks';
+import { alertsMock } from '../mocks';
 import { eventLoggerMock } from '../../../event_log/server/event_logger.mock';
 
 const alertType = {
@@ -48,13 +49,8 @@ describe('Task Runner Factory', () => {
 
   afterAll(() => fakeTimer.restore());
 
-  const savedObjectsClient = savedObjectsClientMock.create();
   const encryptedSavedObjectsPlugin = encryptedSavedObjectsMock.createStart();
-  const services = {
-    log: jest.fn(),
-    callCluster: jest.fn(),
-    savedObjectsClient,
-  };
+  const services = alertsMock.createAlertServices();
 
   const taskRunnerFactoryInitializerParams: jest.Mocked<TaskRunnerContext> = {
     getServices: jest.fn().mockReturnValue(services),

x-pack/plugins/alerting/server/types.ts

@@ -7,14 +7,16 @@
 import { AlertInstance } from './alert_instance';
 import { AlertTypeRegistry as OrigAlertTypeRegistry } from './alert_type_registry';
 import { PluginSetupContract, PluginStartContract } from './plugin';
-import {
-  SavedObjectAttributes,
-  SavedObjectsClientContract,
-  KibanaRequest,
-} from '../../../../src/core/server';
 import { Alert, AlertActionParams, ActionGroup } from '../common';
 import { AlertsClient } from './alerts_client';
 export * from '../common';
+import {
+  IClusterClient,
+  IScopedClusterClient,
+  KibanaRequest,
+  SavedObjectAttributes,
+  SavedObjectsClientContract,
+} from '../../../../src/core/server';
 
 // This will have to remain `any` until we can extend Alert Executors with generics
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -36,10 +38,9 @@ declare module 'src/core/server' {
 }
 
 export interface Services {
-  // This will have to remain `any` until we can extend Alert Services with generics
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  callCluster(path: string, opts: any): Promise<any>;
+  callCluster: IScopedClusterClient['callAsCurrentUser'];
   savedObjectsClient: SavedObjectsClientContract;
+  getScopedCallCluster(clusterClient: IClusterClient): IScopedClusterClient['callAsCurrentUser'];
 }
 
 export interface AlertServices extends Services {

x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts

@@ -20,7 +20,7 @@ const executor = createMetricThresholdExecutor('test') as (opts: {
 }) => Promise<void>;
 
 const services: AlertServicesMock = alertsMock.createAlertServices();
-services.callCluster.mockImplementation((_: string, { body, index }: any) => {
+services.callCluster.mockImplementation(async (_: string, { body, index }: any) => {
   if (index === 'alternatebeat-*') return mocks.changedSourceIdResponse;
   const metric = body.query.bool.filter[1]?.exists.field;
   if (body.aggs.groupings) {

x-pack/test/alerting_api_integration/common/fixtures/plugins/alerts/index.ts

@@ -14,6 +14,7 @@ export default function(kibana: any) {
     require: ['xpack_main', 'actions', 'alerting', 'elasticsearch'],
     name: 'alerts',
     init(server: any) {
+      const clusterClient = server.newPlatform.start.core.elasticsearch.legacy.client;
       server.plugins.xpack_main.registerFeature({
         id: 'alerting',
         name: 'Alerting',
@@ -165,6 +166,22 @@ export default function(kibana: any) {
           } catch (e) {
             callClusterError = e;
           }
+          // Call scoped cluster
+          const callScopedCluster = services.getScopedCallCluster(clusterClient);
+          let callScopedClusterSuccess = false;
+          let callScopedClusterError;
+          try {
+            await callScopedCluster('index', {
+              index: params.callClusterAuthorizationIndex,
+              refresh: 'wait_for',
+              body: {
+                param1: 'test',
+              },
+            });
+            callScopedClusterSuccess = true;
+          } catch (e) {
+            callScopedClusterError = e;
+          }
           // Saved objects client
           let savedObjectsClientSuccess = false;
           let savedObjectsClientError;
@@ -185,6 +202,8 @@ export default function(kibana: any) {
               state: {
                 callClusterSuccess,
                 callClusterError,
+                callScopedClusterSuccess,
+                callScopedClusterError,
                 savedObjectsClientSuccess,
                 savedObjectsClientError,
               },
@@ -376,6 +395,22 @@ export default function(kibana: any) {
           } catch (e) {
             callClusterError = e;
           }
+          // Call scoped cluster
+          const callScopedCluster = services.getScopedCallCluster(clusterClient);
+          let callScopedClusterSuccess = false;
+          let callScopedClusterError;
+          try {
+            await callScopedCluster('index', {
+              index: params.callClusterAuthorizationIndex,
+              refresh: 'wait_for',
+              body: {
+                param1: 'test',
+              },
+            });
+            callScopedClusterSuccess = true;
+          } catch (e) {
+            callScopedClusterError = e;
+          }
           // Saved objects client
           let savedObjectsClientSuccess = false;
           let savedObjectsClientError;
@@ -396,6 +431,8 @@ export default function(kibana: any) {
               state: {
                 callClusterSuccess,
                 callClusterError,
+                callScopedClusterSuccess,
+                callScopedClusterError,
                 savedObjectsClientSuccess,
                 savedObjectsClientError,
               },

x-pack/test/alerting_api_integration/security_and_spaces/tests/actions/execute.ts

@@ -436,11 +436,16 @@ export default function({ getService }: FtrProviderContext) {
               indexedRecord = searchResult.hits.hits[0];
               expect(indexedRecord._source.state).to.eql({
                 callClusterSuccess: false,
+                callScopedClusterSuccess: false,
                 savedObjectsClientSuccess: false,
                 callClusterError: {
                   ...indexedRecord._source.state.callClusterError,
                   statusCode: 403,
                 },
+                callScopedClusterError: {
+                  ...indexedRecord._source.state.callScopedClusterError,
+                  statusCode: 403,
+                },
                 savedObjectsClientError: {
                   ...indexedRecord._source.state.savedObjectsClientError,
                   output: {
@@ -457,6 +462,7 @@ export default function({ getService }: FtrProviderContext) {
               indexedRecord = searchResult.hits.hits[0];
               expect(indexedRecord._source.state).to.eql({
                 callClusterSuccess: true,
+                callScopedClusterSuccess: true,
                 savedObjectsClientSuccess: false,
                 savedObjectsClientError: {
                   ...indexedRecord._source.state.savedObjectsClientError,

x-pack/test/alerting_api_integration/security_and_spaces/tests/alerting/alerts.ts

@@ -469,11 +469,16 @@ instanceStateValue: true
               expect(searchResult.hits.total.value).to.eql(1);
               expect(searchResult.hits.hits[0]._source.state).to.eql({
                 callClusterSuccess: false,
+                callScopedClusterSuccess: false,
                 savedObjectsClientSuccess: false,
                 callClusterError: {
                   ...searchResult.hits.hits[0]._source.state.callClusterError,
                   statusCode: 403,
                 },
+                callScopedClusterError: {
+                  ...searchResult.hits.hits[0]._source.state.callScopedClusterError,
+                  statusCode: 403,
+                },
                 savedObjectsClientError: {
                   ...searchResult.hits.hits[0]._source.state.savedObjectsClientError,
                   output: {
@@ -497,6 +502,7 @@ instanceStateValue: true
               expect(searchResult.hits.total.value).to.eql(1);
               expect(searchResult.hits.hits[0]._source.state).to.eql({
                 callClusterSuccess: true,
+                callScopedClusterSuccess: true,
                 savedObjectsClientSuccess: false,
                 savedObjectsClientError: {
                   ...searchResult.hits.hits[0]._source.state.savedObjectsClientError,
@@ -577,11 +583,16 @@ instanceStateValue: true
               expect(searchResult.hits.total.value).to.eql(1);
               expect(searchResult.hits.hits[0]._source.state).to.eql({
                 callClusterSuccess: false,
+                callScopedClusterSuccess: false,
                 savedObjectsClientSuccess: false,
                 callClusterError: {
                   ...searchResult.hits.hits[0]._source.state.callClusterError,
                   statusCode: 403,
                 },
+                callScopedClusterError: {
+                  ...searchResult.hits.hits[0]._source.state.callScopedClusterError,
+                  statusCode: 403,
+                },
                 savedObjectsClientError: {
                   ...searchResult.hits.hits[0]._source.state.savedObjectsClientError,
                   output: {
@@ -605,6 +616,7 @@ instanceStateValue: true
               expect(searchResult.hits.total.value).to.eql(1);
               expect(searchResult.hits.hits[0]._source.state).to.eql({
                 callClusterSuccess: true,
+                callScopedClusterSuccess: true,
                 savedObjectsClientSuccess: false,
                 savedObjectsClientError: {
                   ...searchResult.hits.hits[0]._source.state.savedObjectsClientError,

x-pack/test/alerting_api_integration/spaces_only/tests/actions/execute.ts

@@ -186,6 +186,7 @@ export default function({ getService }: FtrProviderContext) {
       const indexedRecord = searchResult.hits.hits[0];
       expect(indexedRecord._source.state).to.eql({
         callClusterSuccess: true,
+        callScopedClusterSuccess: true,
         savedObjectsClientSuccess: false,
         savedObjectsClientError: {
           ...indexedRecord._source.state.savedObjectsClientError,

x-pack/test/alerting_api_integration/spaces_only/tests/alerting/alerts_base.ts

@@ -277,6 +277,7 @@ instanceStateValue: true
       )[0];
       expect(alertTestRecord._source.state).to.eql({
         callClusterSuccess: true,
+        callScopedClusterSuccess: true,
         savedObjectsClientSuccess: false,
         savedObjectsClientError: {
           ...alertTestRecord._source.state.savedObjectsClientError,
@@ -332,6 +333,7 @@ instanceStateValue: true
       )[0];
       expect(actionTestRecord._source.state).to.eql({
         callClusterSuccess: true,
+        callScopedClusterSuccess: true,
         savedObjectsClientSuccess: false,
         savedObjectsClientError: {
           ...actionTestRecord._source.state.savedObjectsClientError,