[ML] Anomaly Detection: add ability to clear warning notification from jobs list (#103608)

* wip: adds clear messages endpoint

* wip: clear messages and index new message for clearing

* remove icon from jobs list on clear

* remove unnecessary comments and fix typo

* ensure clear messages has correct permissions

* use cleaner ml context and add type

* only show clear button with canCreateJob and if warning icon in table

* fix types for job message pane

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/ml/common/constants/index_patterns.ts

@@ -11,3 +11,4 @@ export const ML_ANNOTATIONS_INDEX_PATTERN = '.ml-annotations-6';
 
 export const ML_RESULTS_INDEX_PATTERN = '.ml-anomalies-*';
 export const ML_NOTIFICATION_INDEX_PATTERN = '.ml-notifications*';
+export const ML_NOTIFICATION_INDEX_02 = '.ml-notifications-000002';

x-pack/plugins/ml/common/types/anomaly_detection_jobs/summary_job.ts

@@ -46,6 +46,7 @@ export interface AuditMessage {
   highestLevel: string;
   highestLevelText: string;
   text: string;
+  cleared?: boolean;
 }
 
 export type MlSummaryJobs = MlSummaryJob[];

x-pack/plugins/ml/common/types/audit_message.ts

@@ -11,8 +11,10 @@ export interface AuditMessageBase {
   timestamp: number;
   node_name: string;
   text?: string;
+  cleared?: boolean;
 }
 
 export interface JobMessage extends AuditMessageBase {
   job_id: string;
+  clearable?: boolean;
 }

x-pack/plugins/ml/public/application/components/job_messages/job_messages.tsx

@@ -126,7 +126,7 @@ export const JobMessages: FC<JobMessagesProps> = ({
   const defaultSorting = {
     sort: {
       field: 'timestamp' as const,
-      direction: 'asc' as const,
+      direction: 'desc' as const,
     },
   };
 

x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_details.js

@@ -58,7 +58,7 @@ export class JobDetailsUI extends Component {
         </div>
       );
     } else {
-      const { showFullDetails, refreshJobList } = this.props;
+      const { showFullDetails, refreshJobList, showClearButton } = this.props;
 
       const {
         general,
@@ -185,7 +185,13 @@ export class JobDetailsUI extends Component {
           name: i18n.translate('xpack.ml.jobsList.jobDetails.tabs.jobMessagesLabel', {
             defaultMessage: 'Job messages',
           }),
-          content: <JobMessagesPane jobId={job.job_id} />,
+          content: (
+            <JobMessagesPane
+              jobId={job.job_id}
+              refreshJobList={refreshJobList}
+              showClearButton={showClearButton}
+            />
+          ),
         },
       ];
 

x-pack/plugins/ml/public/application/jobs/jobs_list/components/job_details/job_messages_pane.tsx

@@ -6,25 +6,38 @@
  */
 
 import React, { FC, useCallback, useEffect, useState } from 'react';
+import { EuiButton, EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiToolTip } from '@elastic/eui';
 import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
 import { ml } from '../../../../services/ml_api_service';
 import { JobMessages } from '../../../../components/job_messages';
 import { JobMessage } from '../../../../../../common/types/audit_message';
 import { extractErrorMessage } from '../../../../../../common/util/errors';
 import { useToastNotificationService } from '../../../../services/toast_notification_service';
+import { useMlApiContext } from '../../../../contexts/kibana';
+import { checkPermission } from '../../../../capabilities/check_capabilities';
 interface JobMessagesPaneProps {
   jobId: string;
+  showClearButton?: boolean;
   start?: string;
   end?: string;
   actionHandler?: (message: JobMessage) => void;
+  refreshJobList?: () => void;
 }
 
 export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
-  ({ jobId, start, end, actionHandler }) => {
+  ({ jobId, start, end, actionHandler, refreshJobList, showClearButton }) => {
+    const canCreateJob = checkPermission('canCreateJob');
+
     const [messages, setMessages] = useState<JobMessage[]>([]);
     const [isLoading, setIsLoading] = useState(false);
     const [errorMessage, setErrorMessage] = useState('');
+    const [isClearing, setIsClearing] = useState<boolean>(false);
+
     const toastNotificationService = useToastNotificationService();
+    const {
+      jobs: { clearJobAuditMessages },
+    } = useMlApiContext();
 
     const fetchMessages = async () => {
       setIsLoading(true);
@@ -46,18 +59,89 @@ export const JobMessagesPane: FC<JobMessagesPaneProps> = React.memo(
 
     const refreshMessage = useCallback(fetchMessages, [jobId]);
 
+    // Clear messages for last 24hrs and refresh jobs list
+    const clearMessages = useCallback(async () => {
+      setIsClearing(true);
+      try {
+        await clearJobAuditMessages(jobId);
+        setIsClearing(false);
+        if (typeof refreshJobList === 'function') {
+          refreshJobList();
+        }
+      } catch (e) {
+        setIsClearing(false);
+        toastNotificationService.displayErrorToast(
+          e,
+          i18n.translate('xpack.ml.jobMessages.clearJobAuditMessagesErrorTitle', {
+            defaultMessage: 'Error clearing job message warnings and errors',
+          })
+        );
+      }
+    }, [jobId]);
+
     useEffect(() => {
       fetchMessages();
     }, []);
 
+    const disabled = messages.length > 0 && messages[0].clearable === false;
+
+    const clearButton = (
+      <EuiButton
+        size="s"
+        isLoading={isClearing}
+        isDisabled={disabled}
+        onClick={clearMessages}
+        data-test-subj="mlJobMessagesClearButton"
+      >
+        <FormattedMessage
+          id="xpack.ml.jobMessages.clearMessagesLabel"
+          defaultMessage="Clear notifications"
+        />
+      </EuiButton>
+    );
+
     return (
-      <JobMessages
-        refreshMessage={refreshMessage}
-        messages={messages}
-        loading={isLoading}
-        error={errorMessage}
-        actionHandler={actionHandler}
-      />
+      <>
+        <EuiSpacer />
+        <EuiFlexGroup direction="column">
+          {canCreateJob && showClearButton ? (
+            <EuiFlexItem grow={false}>
+              <div>
+                {disabled === true ? (
+                  <EuiToolTip
+                    content={i18n.translate(
+                      'xpack.ml.jobMessages.clearJobAuditMessagesDisabledTooltip',
+                      {
+                        defaultMessage: 'Notification clearing not supported.',
+                      }
+                    )}
+                  >
+                    {clearButton}
+                  </EuiToolTip>
+                ) : (
+                  <EuiToolTip
+                    content={i18n.translate('xpack.ml.jobMessages.clearJobAuditMessagesTooltip', {
+                      defaultMessage:
+                        'Clears warning icon from jobs list for messages produced in the last 24 hours.',
+                    })}
+                  >
+                    {clearButton}
+                  </EuiToolTip>
+                )}
+              </div>
+            </EuiFlexItem>
+          ) : null}
+          <EuiFlexItem grow={false}>
+            <JobMessages
+              refreshMessage={refreshMessage}
+              messages={messages}
+              loading={isLoading}
+              error={errorMessage}
+              actionHandler={actionHandler}
+            />
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </>
     );
   }
 );

x-pack/plugins/ml/public/application/jobs/jobs_list/components/jobs_list_view/jobs_list_view.js

@@ -123,6 +123,9 @@ export class JobsListView extends Component {
       delete itemIdToExpandedRowMap[jobId];
       this.setState({ itemIdToExpandedRowMap });
     } else {
+      // Only show clear notifications button if job has warning icon due to auditMessage
+      const expandedJob = this.state.jobsSummaryList.filter((job) => job.id === jobId);
+      const showClearButton = expandedJob.length > 0 && expandedJob[0].auditMessage !== undefined;
       let itemIdToExpandedRowMap = { ...this.state.itemIdToExpandedRowMap };
 
       if (this.state.fullJobsList[jobId] !== undefined) {
@@ -134,6 +137,7 @@ export class JobsListView extends Component {
             removeYourself={this.removeUpdateFunction}
             showFullDetails={this.props.isManagementTable !== true}
             refreshJobList={this.onRefreshClick}
+            showClearButton={showClearButton}
           />
         );
       } else {
@@ -144,6 +148,7 @@ export class JobsListView extends Component {
             removeYourself={this.removeUpdateFunction}
             showFullDetails={this.props.isManagementTable !== true}
             refreshJobList={this.onRefreshClick}
+            showClearButton={showClearButton}
           />
         );
       }
@@ -167,6 +172,7 @@ export class JobsListView extends Component {
                     removeYourself={this.removeUpdateFunction}
                     showFullDetails={this.props.isManagementTable !== true}
                     refreshJobList={this.onRefreshClick}
+                    showClearButton={showClearButton}
                   />
                 );
               }

x-pack/plugins/ml/public/application/services/ml_api_service/jobs.ts

@@ -160,6 +160,15 @@ export const jobsApiProvider = (httpService: HttpService) => ({
     });
   },
 
+  clearJobAuditMessages(jobId: string) {
+    const body = JSON.stringify({ jobId });
+    return httpService.http<{ success: boolean; latest_cleared: number }>({
+      path: `${ML_BASE_PATH}/job_audit_messages/clear_messages`,
+      method: 'PUT',
+      body,
+    });
+  },
+
   deletingJobTasks() {
     return httpService.http<any>({
       path: `${ML_BASE_PATH}/jobs/deleting_jobs_tasks`,

x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.d.ts

@@ -23,4 +23,5 @@ export function jobAuditMessagesProvider(
     }
   ) => any;
   getAuditMessagesSummary: (jobIds?: string[]) => any;
+  clearJobAuditMessages: (jobId: string) => any;
 };

x-pack/plugins/ml/server/models/job_audit_messages/job_audit_messages.js

@@ -5,7 +5,11 @@
  * 2.0.
  */
 
-import { ML_NOTIFICATION_INDEX_PATTERN } from '../../../common/constants/index_patterns';
+import {
+  ML_NOTIFICATION_INDEX_PATTERN,
+  ML_NOTIFICATION_INDEX_02,
+} from '../../../common/constants/index_patterns';
+import { MESSAGE_LEVEL } from '../../../common/constants/message_levels';
 import moment from 'moment';
 
 const SIZE = 1000;
@@ -35,7 +39,7 @@ const anomalyDetectorTypeFilter = {
   },
 };
 
-export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
+export function jobAuditMessagesProvider({ asInternalUser, asCurrentUser }, mlClient) {
   // search for audit messages,
   // jobId is optional. without it, all jobs will be listed.
   // from is optional and should be a string formatted in ES time units. e.g. 12h, 1d, 7d
@@ -123,7 +127,10 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
 
     let messages = [];
     if (body.hits.total.value > 0) {
-      messages = body.hits.hits.map((hit) => hit._source);
+      messages = body.hits.hits.map((hit) => ({
+        clearable: hit._index === ML_NOTIFICATION_INDEX_02,
+        ...hit._source,
+      }));
     }
     messages = await jobSavedObjectService.filterJobsForSpace(
       'anomaly-detector',
@@ -152,6 +159,11 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
           },
           anomalyDetectorTypeFilter,
         ],
+        must_not: {
+          term: {
+            cleared: true,
+          },
+        },
       },
     };
 
@@ -266,6 +278,61 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
     return jobMessages;
   }
 
+  const clearedTime = new Date().getTime();
+
+  // Sets 'cleared' to true for messages in the last 24hrs and index new message for clear action
+  async function clearJobAuditMessages(jobId) {
+    const newClearedMessage = {
+      job_id: jobId,
+      job_type: 'anomaly_detection',
+      level: MESSAGE_LEVEL.INFO,
+      message: 'Cleared set to true for messages in the last 24hrs.',
+      timestamp: clearedTime,
+    };
+
+    const query = {
+      bool: {
+        filter: [
+          {
+            range: {
+              timestamp: {
+                gte: 'now-24h',
+              },
+            },
+          },
+          {
+            term: {
+              job_id: jobId,
+            },
+          },
+        ],
+      },
+    };
+
+    await Promise.all([
+      asCurrentUser.updateByQuery({
+        index: ML_NOTIFICATION_INDEX_02,
+        ignore_unavailable: true,
+        refresh: true,
+        conflicts: 'proceed',
+        body: {
+          query,
+          script: {
+            source: 'ctx._source.cleared = true',
+            lang: 'painless',
+          },
+        },
+      }),
+      asCurrentUser.index({
+        index: ML_NOTIFICATION_INDEX_02,
+        body: newClearedMessage,
+        refresh: 'wait_for',
+      }),
+    ]);
+
+    return { success: true, last_cleared: clearedTime };
+  }
+
   function levelToText(level) {
     return Object.keys(LEVEL)[Object.values(LEVEL).indexOf(level)];
   }
@@ -273,5 +340,6 @@ export function jobAuditMessagesProvider({ asInternalUser }, mlClient) {
   return {
     getJobAuditMessages,
     getAuditMessagesSummary,
+    clearJobAuditMessages,
   };
 }

x-pack/plugins/ml/server/routes/apidoc.json

@@ -121,7 +121,7 @@
     "JobAuditMessages",
     "GetJobAuditMessages",
     "GetAllJobAuditMessages",
-
+    "ClearJobAuditMessages",
     "JobValidation",
     "EstimateBucketSpan",
     "CalculateModelMemoryLimit",

x-pack/plugins/ml/server/routes/job_audit_messages.ts

@@ -11,6 +11,7 @@ import { jobAuditMessagesProvider } from '../models/job_audit_messages';
 import {
   jobAuditMessagesQuerySchema,
   jobAuditMessagesJobIdSchema,
+  clearJobAuditMessagesBodySchema,
 } from './schemas/job_audit_messages_schema';
 
 /**
@@ -96,4 +97,40 @@ export function jobAuditMessagesRoutes({ router, routeGuard }: RouteInitializati
       }
     )
   );
+
+  /**
+   * @apiGroup JobAuditMessages
+   *
+   * @api {put} /api/ml/job_audit_messages/clear_messages/{jobId} Index annotation
+   * @apiName ClearJobAuditMessages
+   * @apiDescription Clear the job audit messages.
+   *
+   * @apiSchema (body) clearJobAuditMessagesSchema
+   */
+  router.put(
+    {
+      path: '/api/ml/job_audit_messages/clear_messages',
+      validate: {
+        body: clearJobAuditMessagesBodySchema,
+      },
+      options: {
+        tags: ['access:ml:canCreateJob'],
+      },
+    },
+    routeGuard.fullLicenseAPIGuard(
+      async ({ client, mlClient, request, response, jobSavedObjectService }) => {
+        try {
+          const { clearJobAuditMessages } = jobAuditMessagesProvider(client, mlClient);
+          const { jobId } = request.body;
+          const resp = await clearJobAuditMessages(jobId);
+
+          return response.ok({
+            body: resp,
+          });
+        } catch (e) {
+          return response.customError(wrapError(e));
+        }
+      }
+    )
+  );
 }

x-pack/plugins/ml/server/routes/schemas/job_audit_messages_schema.ts

@@ -17,3 +17,7 @@ export const jobAuditMessagesQuerySchema = schema.object({
   start: schema.maybe(schema.string()),
   end: schema.maybe(schema.string()),
 });
+
+export const clearJobAuditMessagesBodySchema = schema.object({
+  jobId: schema.string(),
+});