[Security Solutions][Detection Engine] Removes duplicate API calls (#88420)

## Summary

This removes some duplicate API calls to reduce pressure on the backend and speed up querying times within the application for the front end. This fixes some of the issues of https://github.com/elastic/kibana/issues/82327, but there are several performance improvements that are going to be needed to help reduce the slowness when you have a system under a lot of pressure.

So far this removes duplication for these API calls when you are on the manage detection rules page:

```ts
api/detection_engine/rules/_find
api/detection_engine/rules/_find_statuses
api/detection_engine/tags
```

<img width="2465" alt="Screen Shot 2021-01-14 at 3 53 21 PM" src="https://user-images.githubusercontent.com/1151048/104662295-c031e080-5687-11eb-92d7-18b9ad355646.png">

* This hides the tags and searches while the page is loading to avoid duplicate calls when the pre-packaged rules counts come back
* This untangles the refetchRules from the refetchPrePackagedRulesStatus as two separate calls to avoid issues we have with re-rendering and re-calling the backend.
 
### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_query.tsx

@@ -9,7 +9,7 @@ import React, { SetStateAction, useEffect, useState } from 'react';
 import { fetchQueryAlerts } from './api';
 import { AlertSearchResponse } from './types';
 
-type Func = () => void;
+type Func = () => Promise<void>;
 
 export interface ReturnQueryAlerts<Hit, Aggs> {
   loading: boolean;

x-pack/plugins/security_solution/public/detections/containers/detection_engine/alerts/use_signal_index.tsx

@@ -11,7 +11,7 @@ import { createSignalIndex, getSignalIndex } from './api';
 import * as i18n from './translations';
 import { isSecurityAppError } from '../../../../common/utils/api';
 
-type Func = () => void;
+type Func = () => Promise<void>;
 
 export interface ReturnSignalIndex {
   loading: boolean;

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/api.ts

@@ -120,9 +120,9 @@ export const fetchRules = async ({
     ...showElasticRuleFilter,
   ].join(' AND ');
 
-  const tags = [
+  const tags = filterOptions.tags
-    ...(filterOptions.tags?.map((t) => `alert.attributes.tags: "${t.replace(/"/g, '\\"')}"`) ?? []),
+    .map((t) => `alert.attributes.tags: "${t.replace(/"/g, '\\"')}"`)
-  ].join(' AND ');
+    .join(' AND ');
 
   const filterString =
     filtersWithoutTags !== '' && tags !== ''

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/types.ts

@@ -177,9 +177,9 @@ export interface FilterOptions {
   filter: string;
   sortField: RulesSortingFields;
   sortOrder: SortOrder;
-  showCustomRules?: boolean;
+  showCustomRules: boolean;
-  showElasticRules?: boolean;
+  showElasticRules: boolean;
-  tags?: string[];
+  tags: string[];
 }
 
 export interface FetchRulesResponse {

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_pre_packaged_rules.tsx

@@ -20,7 +20,7 @@ import {
   getPrePackagedTimelineStatus,
 } from '../../../pages/detection_engine/rules/helpers';
 
-type Func = () => void;
+type Func = () => Promise<void>;
 export type CreatePreBuiltRules = () => Promise<boolean>;
 
 interface ReturnPrePackagedTimelines {

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rule_status.tsx

@@ -113,9 +113,11 @@ export const useRulesStatuses = (rules: Rules): ReturnRulesStatuses => {
         setLoading(false);
       }
     };
-    if (rules != null && rules.length > 0) {
+
+    if (rules.length > 0) {
       fetchData(rules.map((r) => r.id));
     }
+
     return () => {
       isSubscribed = false;
       abortCtrl.abort();

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rules.test.tsx

@@ -27,6 +27,9 @@ describe('useRules', () => {
             filter: '',
             sortField: 'created_at',
             sortOrder: 'desc',
+            tags: [],
+            showCustomRules: false,
+            showElasticRules: false,
           },
         })
       );
@@ -48,6 +51,9 @@ describe('useRules', () => {
             filter: '',
             sortField: 'created_at',
             sortOrder: 'desc',
+            tags: [],
+            showCustomRules: false,
+            showElasticRules: false,
           },
         })
       );
@@ -153,6 +159,9 @@ describe('useRules', () => {
             filter: '',
             sortField: 'created_at',
             sortOrder: 'desc',
+            tags: [],
+            showCustomRules: false,
+            showElasticRules: false,
           },
         })
       );
@@ -182,6 +191,9 @@ describe('useRules', () => {
               filter: '',
               sortField: 'created_at',
               sortOrder: 'desc',
+              tags: [],
+              showCustomRules: false,
+              showElasticRules: false,
             },
           },
         }
@@ -198,6 +210,9 @@ describe('useRules', () => {
           filter: 'hello world',
           sortField: 'created_at',
           sortOrder: 'desc',
+          tags: [],
+          showCustomRules: false,
+          showElasticRules: false,
         },
       });
       await waitForNextUpdate();

x-pack/plugins/security_solution/public/detections/containers/detection_engine/rules/use_rules.tsx

@@ -4,7 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { noop } from 'lodash/fp';
 import { useEffect, useState, useRef } from 'react';
 
 import { FetchRulesResponse, FilterOptions, PaginationOptions, Rule } from './types';
@@ -12,16 +11,11 @@ import { errorToToaster, useStateToaster } from '../../../../common/components/t
 import { fetchRules } from './api';
 import * as i18n from './translations';
 
-export type ReturnRules = [
+export type ReturnRules = [boolean, FetchRulesResponse | null, () => Promise<void>];
-  boolean,
-  FetchRulesResponse | null,
-  (refreshPrePackagedRule?: boolean) => void
-];
 
 export interface UseRules {
   pagination: PaginationOptions;
   filterOptions: FilterOptions;
-  refetchPrePackagedRulesStatus?: () => void;
   dispatchRulesInReducer?: (rules: Rule[], pagination: Partial<PaginationOptions>) => void;
 }
 
@@ -34,20 +28,19 @@ export interface UseRules {
 export const useRules = ({
   pagination,
   filterOptions,
-  refetchPrePackagedRulesStatus,
   dispatchRulesInReducer,
 }: UseRules): ReturnRules => {
   const [rules, setRules] = useState<FetchRulesResponse | null>(null);
-  const reFetchRules = useRef<(refreshPrePackagedRule?: boolean) => void>(noop);
+  const reFetchRules = useRef<() => Promise<void>>(() => Promise.resolve());
   const [loading, setLoading] = useState(true);
   const [, dispatchToaster] = useStateToaster();
 
-  const filterTags = filterOptions.tags?.sort().join();
+  const filterTags = filterOptions.tags.sort().join();
   useEffect(() => {
     let isSubscribed = true;
     const abortCtrl = new AbortController();
 
-    async function fetchData() {
+    const fetchData = async () => {
       try {
         setLoading(true);
         const fetchRulesResult = await fetchRules({
@@ -77,15 +70,10 @@ export const useRules = ({
       if (isSubscribed) {
         setLoading(false);
       }
-    }
+    };
 
     fetchData();
-    reFetchRules.current = (refreshPrePackagedRule: boolean = false) => {
+    reFetchRules.current = (): Promise<void> => fetchData();
-      fetchData();
-      if (refreshPrePackagedRule && refetchPrePackagedRulesStatus != null) {
-        refetchPrePackagedRulesStatus();
-      }
-    };
     return () => {
       isSubscribed = false;
       abortCtrl.abort();
@@ -100,7 +88,6 @@ export const useRules = ({
     filterTags,
     filterOptions.showCustomRules,
     filterOptions.showElasticRules,
-    refetchPrePackagedRulesStatus,
   ]);
 
   return [loading, rules, reFetchRules.current];

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/batch_actions.tsx

@@ -27,7 +27,8 @@ interface GetBatchItems {
   hasMlPermissions: boolean;
   hasActionsPrivileges: boolean;
   loadingRuleIds: string[];
-  reFetchRules: (refreshPrePackagedRule?: boolean) => void;
+  reFetchRules: () => Promise<void>;
+  refetchPrePackagedRulesStatus: () => Promise<void>;
   rules: Rule[];
   selectedRuleIds: string[];
 }
@@ -39,17 +40,18 @@ export const getBatchItems = ({
   hasMlPermissions,
   loadingRuleIds,
   reFetchRules,
+  refetchPrePackagedRulesStatus,
   rules,
   selectedRuleIds,
   hasActionsPrivileges,
 }: GetBatchItems) => {
-  const selectedRules = selectedRuleIds.reduce((acc, id) => {
+  const selectedRules = selectedRuleIds.reduce<Record<string, Rule>>((acc, id) => {
     const found = rules.find((r) => r.id === id);
     if (found != null) {
       return { [id]: found, ...acc };
     }
     return acc;
-  }, {} as Record<string, Rule>);
+  }, {});
 
   const containsEnabled = selectedRuleIds.some((id) => selectedRules[id]?.enabled ?? false);
   const containsDisabled = selectedRuleIds.some((id) => !selectedRules[id]?.enabled ?? false);
@@ -139,7 +141,8 @@ export const getBatchItems = ({
           dispatch,
           dispatchToaster
         );
-        reFetchRules(true);
+        await reFetchRules();
+        await refetchPrePackagedRulesStatus();
       }}
     >
       <EuiToolTip
@@ -158,7 +161,8 @@ export const getBatchItems = ({
       onClick={async () => {
         closePopover();
         await deleteRulesAction(selectedRuleIds, dispatch, dispatchToaster);
-        reFetchRules(true);
+        await reFetchRules();
+        await refetchPrePackagedRulesStatus();
       }}
     >
       {i18n.BATCH_ACTION_DELETE_SELECTED}

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.test.tsx

@@ -27,6 +27,7 @@ describe('AllRulesTable Columns', () => {
     const dispatch = jest.fn();
     const dispatchToaster = jest.fn();
     const reFetchRules = jest.fn();
+    const refetchPrePackagedRulesStatus = jest.fn();
 
     beforeEach(() => {
       results = [];
@@ -53,6 +54,7 @@ describe('AllRulesTable Columns', () => {
         dispatchToaster,
         history,
         reFetchRules,
+        refetchPrePackagedRulesStatus,
         true
       )[1];
       await duplicateRulesActionObject.onClick(rule);
@@ -75,6 +77,7 @@ describe('AllRulesTable Columns', () => {
         dispatchToaster,
         history,
         reFetchRules,
+        refetchPrePackagedRulesStatus,
         true
       )[3];
       await deleteRulesActionObject.onClick(rule);

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/columns.tsx

@@ -43,7 +43,8 @@ export const getActions = (
   dispatch: React.Dispatch<Action>,
   dispatchToaster: Dispatch<ActionToaster>,
   history: H.History,
-  reFetchRules: (refreshPrePackagedRule?: boolean) => void,
+  reFetchRules: () => Promise<void>,
+  refetchPrePackagedRulesStatus: () => Promise<void>,
   actionsPrivileges:
     | boolean
     | Readonly<{
@@ -77,7 +78,8 @@ export const getActions = (
     enabled: (rowItem: Rule) => canEditRuleWithActions(rowItem, actionsPrivileges),
     onClick: async (rowItem: Rule) => {
       await duplicateRulesAction([rowItem], [rowItem.id], dispatch, dispatchToaster);
-      await reFetchRules(true);
+      await reFetchRules();
+      await refetchPrePackagedRulesStatus();
     },
   },
   {
@@ -95,7 +97,8 @@ export const getActions = (
     name: i18n.DELETE_RULE,
     onClick: async (rowItem: Rule) => {
       await deleteRulesAction([rowItem.id], dispatch, dispatchToaster);
-      await reFetchRules(true);
+      await reFetchRules();
+      await refetchPrePackagedRulesStatus();
     },
   },
 ];
@@ -115,7 +118,8 @@ interface GetColumns {
   hasMlPermissions: boolean;
   hasNoPermissions: boolean;
   loadingRuleIds: string[];
-  reFetchRules: (refreshPrePackagedRule?: boolean) => void;
+  reFetchRules: () => Promise<void>;
+  refetchPrePackagedRulesStatus: () => Promise<void>;
   hasReadActionsPrivileges:
     | boolean
     | Readonly<{
@@ -132,6 +136,7 @@ export const getColumns = ({
   hasNoPermissions,
   loadingRuleIds,
   reFetchRules,
+  refetchPrePackagedRulesStatus,
   hasReadActionsPrivileges,
 }: GetColumns): RulesColumns[] => {
   const cols: RulesColumns[] = [
@@ -279,6 +284,7 @@ export const getColumns = ({
         dispatchToaster,
         history,
         reFetchRules,
+        refetchPrePackagedRulesStatus,
         hasReadActionsPrivileges
       ),
       width: '40px',

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/exceptions/exceptions_table.tsx

@@ -36,7 +36,7 @@ import { patchRule } from '../../../../../containers/detection_engine/rules/api'
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 const MyEuiBasicTable = styled(EuiBasicTable as any)`` as any;
 
-export type Func = () => void;
+export type Func = () => Promise<void>;
 export interface ExceptionListFilter {
   name?: string | null;
   list_id?: string | null;

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/index.tsx

@@ -20,12 +20,12 @@ interface AllRulesProps {
   hasNoPermissions: boolean;
   loading: boolean;
   loadingCreatePrePackagedRules: boolean;
-  refetchPrePackagedRulesStatus: () => void;
+  refetchPrePackagedRulesStatus: () => Promise<void>;
   rulesCustomInstalled: number | null;
   rulesInstalled: number | null;
   rulesNotInstalled: number | null;
   rulesNotUpdated: number | null;
-  setRefreshRulesData: (refreshRule: (refreshPrePackagedRule?: boolean) => void) => void;
+  setRefreshRulesData: (refreshRule: () => Promise<void>) => void;
 }
 
 export enum AllRulesTabs {

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/reducer.test.ts

@@ -14,6 +14,9 @@ const initialState: State = {
     filter: '',
     sortField: 'enabled',
     sortOrder: 'desc',
+    tags: [],
+    showCustomRules: false,
+    showElasticRules: false,
   },
   loadingRuleIds: [],
   loadingRulesAction: null,
@@ -193,6 +196,9 @@ describe('allRulesReducer', () => {
         filter: 'host.name:*',
         sortField: 'enabled',
         sortOrder: 'desc',
+        tags: [],
+        showCustomRules: false,
+        showElasticRules: false,
       };
       const { filterOptions, pagination } = reducer(initialState, {
         type: 'updateFilterOptions',

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/all/rules_tables.tsx

@@ -60,6 +60,9 @@ const initialState: State = {
     filter: '',
     sortField: INITIAL_SORT_FIELD,
     sortOrder: 'desc',
+    tags: [],
+    showCustomRules: false,
+    showElasticRules: false,
   },
   loadingRuleIds: [],
   loadingRulesAction: null,
@@ -82,12 +85,12 @@ interface RulesTableProps {
   hasNoPermissions: boolean;
   loading: boolean;
   loadingCreatePrePackagedRules: boolean;
-  refetchPrePackagedRulesStatus: () => void;
+  refetchPrePackagedRulesStatus: () => Promise<void>;
   rulesCustomInstalled: number | null;
   rulesInstalled: number | null;
   rulesNotInstalled: number | null;
   rulesNotUpdated: number | null;
-  setRefreshRulesData: (refreshRule: (refreshPrePackagedRule?: boolean) => void) => void;
+  setRefreshRulesData: (refreshRule: () => Promise<void>) => void;
   selectedTab: AllRulesTabs;
 }
 
@@ -183,10 +186,9 @@ export const RulesTables = React.memo<RulesTableProps>(
       });
     }, []);
 
-    const [isLoadingRules, , reFetchRulesData] = useRules({
+    const [isLoadingRules, , reFetchRules] = useRules({
       pagination,
       filterOptions,
-      refetchPrePackagedRulesStatus,
       dispatchRulesInReducer: setRules,
     });
 
@@ -220,7 +222,8 @@ export const RulesTables = React.memo<RulesTableProps>(
           hasActionsPrivileges,
           loadingRuleIds,
           selectedRuleIds,
-          reFetchRules: reFetchRulesData,
+          reFetchRules,
+          refetchPrePackagedRulesStatus,
           rules,
         });
       },
@@ -229,7 +232,8 @@ export const RulesTables = React.memo<RulesTableProps>(
         dispatchToaster,
         hasMlPermissions,
         loadingRuleIds,
-        reFetchRulesData,
+        reFetchRules,
+        refetchPrePackagedRulesStatus,
         rules,
         selectedRuleIds,
         hasActionsPrivileges,
@@ -273,19 +277,22 @@ export const RulesTables = React.memo<RulesTableProps>(
           (loadingRulesAction === 'enable' || loadingRulesAction === 'disable')
             ? loadingRuleIds
             : [],
-        reFetchRules: reFetchRulesData,
+        reFetchRules,
+        refetchPrePackagedRulesStatus,
         hasReadActionsPrivileges: hasActionsPrivileges,
       });
-      // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [
       dispatch,
       dispatchToaster,
       formatUrl,
+      refetchPrePackagedRulesStatus,
+      hasActionsPrivileges,
+      hasNoPermissions,
       hasMlPermissions,
       history,
       loadingRuleIds,
       loadingRulesAction,
-      reFetchRulesData,
+      reFetchRules,
     ]);
 
     const monitoringColumns = useMemo(() => getMonitoringColumns(history, formatUrl), [
@@ -294,10 +301,8 @@ export const RulesTables = React.memo<RulesTableProps>(
     ]);
 
     useEffect(() => {
-      if (reFetchRulesData != null) {
+      setRefreshRulesData(reFetchRules);
-        setRefreshRulesData(reFetchRulesData);
+    }, [reFetchRules, setRefreshRulesData]);
-      }
-    }, [reFetchRulesData, setRefreshRulesData]);
 
     useEffect(() => {
       if (initLoading && !loading && !isLoadingRules && !isLoadingRulesStatuses) {
@@ -306,11 +311,12 @@ export const RulesTables = React.memo<RulesTableProps>(
     }, [initLoading, loading, isLoadingRules, isLoadingRulesStatuses]);
 
     const handleCreatePrePackagedRules = useCallback(async () => {
-      if (createPrePackagedRules != null && reFetchRulesData != null) {
+      if (createPrePackagedRules != null) {
         await createPrePackagedRules();
-        reFetchRulesData(true);
+        await reFetchRules();
+        await refetchPrePackagedRulesStatus();
       }
-    }, [createPrePackagedRules, reFetchRulesData]);
+    }, [createPrePackagedRules, reFetchRules, refetchPrePackagedRulesStatus]);
 
     const euiBasicTableSelectionProps = useMemo(
       () => ({
@@ -343,12 +349,13 @@ export const RulesTables = React.memo<RulesTableProps>(
       return false;
     }, [loadingRuleIds, loadingRulesAction]);
 
-    const handleRefreshData = useCallback((): void => {
+    const handleRefreshData = useCallback(async (): Promise<void> => {
-      if (reFetchRulesData != null && !isLoadingAnActionOnRule) {
+      if (!isLoadingAnActionOnRule) {
-        reFetchRulesData(true);
+        await reFetchRules();
+        await refetchPrePackagedRulesStatus();
         setLastRefreshDate();
       }
-    }, [reFetchRulesData, isLoadingAnActionOnRule, setLastRefreshDate]);
+    }, [reFetchRules, isLoadingAnActionOnRule, setLastRefreshDate, refetchPrePackagedRulesStatus]);
 
     const handleResetIdleTimer = useCallback((): void => {
       if (isRefreshOn) {
@@ -458,12 +465,14 @@ export const RulesTables = React.memo<RulesTableProps>(
                 />
               }
             >
-              <RulesTableFilters
+              {shouldShowRulesTable && (
-                onFilterChanged={onFilterChangedCallback}
+                <RulesTableFilters
-                rulesCustomInstalled={rulesCustomInstalled}
+                  onFilterChanged={onFilterChangedCallback}
-                rulesInstalled={rulesInstalled}
+                  rulesCustomInstalled={rulesCustomInstalled}
-                currentFilterTags={filterOptions.tags ?? []}
+                  rulesInstalled={rulesInstalled}
-              />
+                  currentFilterTags={filterOptions.tags}
+                />
+              )}
             </HeaderSection>
 
             {isLoadingAnActionOnRule && !initLoading && (

x-pack/plugins/security_solution/public/detections/pages/detection_engine/rules/index.tsx

@@ -35,7 +35,7 @@ import { SecurityPageName } from '../../../../app/types';
 import { LinkButton } from '../../../../common/components/links';
 import { useFormatUrl } from '../../../../common/components/link_to';
 
-type Func = (refreshPrePackagedRule?: boolean) => void;
+type Func = () => Promise<void>;
 
 const RulesPageComponent: React.FC = () => {
   const history = useHistory();
@@ -94,20 +94,22 @@ const RulesPageComponent: React.FC = () => {
 
   const handleRefreshRules = useCallback(async () => {
     if (refreshRulesData.current != null) {
-      refreshRulesData.current(true);
+      await refreshRulesData.current();
     }
   }, [refreshRulesData]);
 
   const handleCreatePrePackagedRules = useCallback(async () => {
     if (createPrePackagedRules != null) {
       await createPrePackagedRules();
-      handleRefreshRules();
+      return handleRefreshRules();
     }
   }, [createPrePackagedRules, handleRefreshRules]);
 
   const handleRefetchPrePackagedRulesStatus = useCallback(() => {
     if (refetchPrePackagedRulesStatus != null) {
-      refetchPrePackagedRulesStatus();
+      return refetchPrePackagedRulesStatus();
+    } else {
+      return Promise.resolve();
     }
   }, [refetchPrePackagedRulesStatus]);