[Monitoring] Remove deprecated watcher-based cluster alerts (#85047)

* First draft

* Update to use actual API

* Remove this file

* Update translation key

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/monitoring/public/alerts/lib/alerts_toast.tsx

@@ -11,9 +11,10 @@ import { EuiSpacer, EuiLink } from '@elastic/eui';
 import { Legacy } from '../../legacy_shims';
 import { toMountPoint } from '../../../../../../src/plugins/kibana_react/public';
 
-export interface AlertingFrameworkHealth {
-  isSufficientlySecure: boolean;
-  hasPermanentEncryptionKey: boolean;
+export interface EnableAlertResponse {
+  isSufficientlySecure?: boolean;
+  hasPermanentEncryptionKey?: boolean;
+  disabledWatcherClusterAlerts?: boolean;
 }
 
 const showTlsAndEncryptionError = () => {
@@ -48,18 +49,48 @@ const showTlsAndEncryptionError = () => {
   });
 };
 
-export const showSecurityToast = (alertingHealth: AlertingFrameworkHealth) => {
-  const { isSufficientlySecure, hasPermanentEncryptionKey } = alertingHealth;
+const showUnableToDisableWatcherClusterAlertsError = () => {
+  const { ELASTIC_WEBSITE_URL, DOC_LINK_VERSION } = Legacy.shims.docLinks;
 
-  if (
-    Array.isArray(alertingHealth) ||
-    (!alertingHealth.hasOwnProperty('isSufficientlySecure') &&
-      !alertingHealth.hasOwnProperty('hasPermanentEncryptionKey'))
-  ) {
-    return;
-  }
+  Legacy.shims.toastNotifications.addWarning({
+    title: toMountPoint(
+      <FormattedMessage
+        id="xpack.monitoring.healthCheck.unableToDisableWatches.title"
+        defaultMessage="Legacy cluster alerts still active"
+      />
+    ),
+    text: toMountPoint(
+      <div>
+        <p>
+          {i18n.translate('xpack.monitoring.healthCheck.unableToDisableWatches.text', {
+            defaultMessage: `We failed to remove legacy cluster alerts. Please check the Kibana server log for more details, or try again later.`,
+          })}
+        </p>
+        <EuiSpacer size="xs" />
+        <EuiLink
+          href={`${ELASTIC_WEBSITE_URL}guide/en/kibana/${DOC_LINK_VERSION}/alert-action-settings-kb.html#general-alert-action-settings`}
+          external
+          target="_blank"
+        >
+          {i18n.translate('xpack.monitoring.healthCheck.unableToDisableWatches.action', {
+            defaultMessage: 'Learn more.',
+          })}
+        </EuiLink>
+      </div>
+    ),
+  });
+};
 
-  if (!isSufficientlySecure || !hasPermanentEncryptionKey) {
+export const showAlertsToast = (response: EnableAlertResponse) => {
+  const {
+    isSufficientlySecure,
+    hasPermanentEncryptionKey,
+    disabledWatcherClusterAlerts,
+  } = response;
+
+  if (isSufficientlySecure === false || hasPermanentEncryptionKey === false) {
     showTlsAndEncryptionError();
+  } else if (disabledWatcherClusterAlerts === false) {
+    showUnableToDisableWatcherClusterAlertsError();
   }
 };

x-pack/plugins/monitoring/public/services/clusters.js

@@ -8,7 +8,7 @@ import { ajaxErrorHandlersProvider } from '../lib/ajax_error_handler';
 import { Legacy } from '../legacy_shims';
 import { STANDALONE_CLUSTER_CLUSTER_UUID } from '../../common/constants';
 import { showInternalMonitoringToast } from '../lib/internal_monitoring_toasts';
-import { showSecurityToast } from '../alerts/lib/security_toasts';
+import { showAlertsToast } from '../alerts/lib/alerts_toast';
 
 function formatClusters(clusters) {
   return clusters.map(formatCluster);
@@ -94,7 +94,7 @@ export function monitoringClustersProvider($injector) {
       if (clusters.length) {
         try {
           const [{ data }] = await Promise.all([ensureAlertsEnabled(), ensureMetricbeatEnabled()]);
-          showSecurityToast(data);
+          showAlertsToast(data);
           once = true;
         } catch (_err) {
           // Intentionally swallow the error as this will retry the next page load

x-pack/plugins/monitoring/server/es_client/instantiate_client.ts

@@ -7,6 +7,7 @@ import { ConfigOptions } from 'elasticsearch';
 import { Logger, ILegacyCustomClusterClient } from 'kibana/server';
 // @ts-ignore
 import { monitoringBulk } from '../kibana_monitoring/lib/monitoring_bulk';
+import { monitoringEndpointDisableWatches } from './monitoring_endpoint_disable_watches';
 import { MonitoringElasticsearchConfig } from '../config';
 
 /* Provide a dedicated Elasticsearch client for Monitoring
@@ -28,7 +29,7 @@ export function instantiateClient(
   const isMonitoringCluster = hasMonitoringCluster(elasticsearchConfig);
   const cluster = createClient('monitoring', {
     ...(isMonitoringCluster ? elasticsearchConfig : {}),
-    plugins: [monitoringBulk],
+    plugins: [monitoringBulk, monitoringEndpointDisableWatches],
     logQueries: Boolean(elasticsearchConfig.logQueries),
   } as ESClusterConfig);
 

x-pack/plugins/monitoring/server/es_client/monitoring_endpoint_disable_watches.ts

@@ -0,0 +1,20 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export function monitoringEndpointDisableWatches(Client: any, _config: any, components: any) {
+  const ca = components.clientAction.factory;
+  Client.prototype.monitoring = components.clientAction.namespaceFactory();
+  const monitoring = Client.prototype.monitoring.prototype;
+  monitoring.disableWatches = ca({
+    params: {},
+    urls: [
+      {
+        fmt: '_monitoring/migrate/alerts',
+      },
+    ],
+    method: 'POST',
+  });
+}

x-pack/plugins/monitoring/server/lib/alerts/disable_watcher_cluster_alerts.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { Logger } from 'kibana/server';
+import { LegacyAPICaller } from 'src/core/server';
+
+interface DisableWatchesResponse {
+  exporters: Array<
+    Array<{
+      name: string;
+      type: string;
+      migration_complete: boolean;
+      reason?: {
+        type: string;
+        reason: string;
+      };
+    }>
+  >;
+}
+
+async function callMigrationApi(callCluster: LegacyAPICaller) {
+  return await callCluster('monitoring.disableWatches');
+}
+
+export async function disableWatcherClusterAlerts(callCluster: LegacyAPICaller, logger: Logger) {
+  const response: DisableWatchesResponse = await callMigrationApi(callCluster);
+  if (!response || response.exporters.length === 0) {
+    return true;
+  }
+  const list = response.exporters[0];
+  if (list.length === 0) {
+    return true;
+  }
+
+  let removedAll = true;
+  for (const exporter of list) {
+    if (!exporter.migration_complete) {
+      if (exporter.reason) {
+        logger.warn(
+          `Unable to remove exporter type=${exporter.type} and name=${exporter.name} because ${exporter.reason.type}: ${exporter.reason.reason}`
+        );
+        removedAll = false;
+      }
+    }
+  }
+  return removedAll;
+}

x-pack/plugins/monitoring/server/plugin.ts

@@ -211,9 +211,11 @@ export class Plugin {
 
       this.registerPluginInUI(plugins);
       requireUIRoutes(this.monitoringCore, {
+        cluster,
         router,
         licenseService: this.licenseService,
         encryptedSavedObjects: plugins.encryptedSavedObjects,
+        logger: this.log,
       });
       initInfraSource(config, plugins.infra);
     }

x-pack/plugins/monitoring/server/routes/api/v1/alerts/enable.ts

@@ -11,6 +11,8 @@ import { RouteDependencies } from '../../../../types';
 import { ALERT_ACTION_TYPE_LOG } from '../../../../../common/constants';
 import { ActionResult } from '../../../../../../actions/common';
 import { AlertingSecurity } from '../../../../lib/elasticsearch/verify_alerting_security';
+import { disableWatcherClusterAlerts } from '../../../../lib/alerts/disable_watcher_cluster_alerts';
+import { Alert, AlertTypeParams } from '../../../../../../alerts/common';
 
 const DEFAULT_SERVER_LOG_NAME = 'Monitoring: Write to Kibana log';
 
@@ -20,7 +22,7 @@ export function enableAlertsRoute(_server: unknown, npRoute: RouteDependencies)
       path: '/api/monitoring/v1/alerts/enable',
       validate: false,
     },
-    async (context, _request, response) => {
+    async (context, request, response) => {
       try {
         const alerts = AlertsFactory.getAll().filter((a) => a.isEnabled(npRoute.licenseService));
 
@@ -75,12 +77,19 @@ export function enableAlertsRoute(_server: unknown, npRoute: RouteDependencies)
           },
         ];
 
-        const createdAlerts = await Promise.all(
-          alerts.map(
-            async (alert) => await alert.createIfDoesNotExist(alertsClient, actionsClient, actions)
-          )
+        let createdAlerts: Array<Alert<AlertTypeParams>> = [];
+        const disabledWatcherClusterAlerts = await disableWatcherClusterAlerts(
+          npRoute.cluster.asScoped(request).callAsCurrentUser,
+          npRoute.logger
         );
-        return response.ok({ body: createdAlerts });
+
+        if (disabledWatcherClusterAlerts) {
+          createdAlerts = await Promise.all(
+            alerts.map((alert) => alert.createIfDoesNotExist(alertsClient, actionsClient, actions))
+          );
+        }
+
+        return response.ok({ body: { createdAlerts, disabledWatcherClusterAlerts } });
       } catch (err) {
         throw handleError(err);
       }

x-pack/plugins/monitoring/server/types.ts

@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { Observable } from 'rxjs';
-import { IRouter, ILegacyClusterClient, Logger } from 'kibana/server';
+import { IRouter, ILegacyClusterClient, Logger, ILegacyCustomClusterClient } from 'kibana/server';
 import { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 import { LicenseFeature, ILicense } from '../../licensing/server';
 import { PluginStartContract as ActionsPluginsStartContact } from '../../actions/server';
@@ -53,9 +53,11 @@ export interface MonitoringCoreConfig {
 }
 
 export interface RouteDependencies {
+  cluster: ILegacyCustomClusterClient;
   router: IRouter;
   licenseService: MonitoringLicenseService;
   encryptedSavedObjects?: EncryptedSavedObjectsPluginSetup;
+  logger: Logger;
 }
 
 export interface MonitoringCore {