Exceptions export duplicates (#116698) (#117197)

## Summary

Addresses https://github.com/elastic/kibana/issues/116329

Removes duplicate exception lists on rule export when multiple rules reference the same list.

Co-authored-by: Yara Tercero <yctercero@users.noreply.github.com>

x-pack/plugins/lists/common/schemas/response/exception_list_schema.mock.ts

@@ -16,6 +16,7 @@ import {
 import {
   DATE_NOW,
   DESCRIPTION,
+  DETECTION_TYPE,
   ELASTIC_USER,
   ENDPOINT_TYPE,
   IMMUTABLE,
@@ -48,6 +49,26 @@ export const getExceptionListSchemaMock = (): ExceptionListSchema => ({
   version: VERSION,
 });
 
+export const getDetectionsExceptionListSchemaMock = (): ExceptionListSchema => ({
+  _version: _VERSION,
+  created_at: DATE_NOW,
+  created_by: USER,
+  description: DESCRIPTION,
+  id: '1',
+  immutable: IMMUTABLE,
+  list_id: 'exception_list_id',
+  meta: META,
+  name: 'Sample Exception List',
+  namespace_type: 'single',
+  os_types: ['linux'],
+  tags: ['user added string for a tag', 'malware'],
+  tie_breaker_id: TIE_BREAKER,
+  type: DETECTION_TYPE,
+  updated_at: DATE_NOW,
+  updated_by: 'user_name',
+  version: VERSION,
+});
+
 export const getTrustedAppsListSchemaMock = (): ExceptionListSchema => {
   return {
     ...getExceptionListSchemaMock(),

x-pack/plugins/lists/server/scripts/check_env_variables.sh

@@ -30,13 +30,3 @@ if [ -z "${KIBANA_URL}" ]; then
   echo "Set KIBANA_URL in your environment"
   exit 1
 fi
-
-if [ -z "${TASK_MANAGER_INDEX}" ]; then
-  echo "Set TASK_MANAGER_INDEX in your environment"
-  exit 1
-fi
-
-if [ -z "${KIBANA_INDEX}" ]; then
-  echo "Set KIBANA_INDEX in your environment"
-  exit 1
-fi

x-pack/plugins/lists/server/services/exception_lists/exception_list_client.mock.ts

@@ -11,6 +11,7 @@ import { getFoundExceptionListSchemaMock } from '../../../common/schemas/respons
 import { getFoundExceptionListItemSchemaMock } from '../../../common/schemas/response/found_exception_list_item_schema.mock';
 import { getExceptionListItemSchemaMock } from '../../../common/schemas/response/exception_list_item_schema.mock';
 import {
+  getDetectionsExceptionListSchemaMock,
   getExceptionListSchemaMock,
   getTrustedAppsListSchemaMock,
 } from '../../../common/schemas/response/exception_list_schema.mock';
@@ -31,10 +32,12 @@ export class ExceptionListClientMock extends ExceptionListClient {
   public createTrustedAppsList = jest.fn().mockResolvedValue(getTrustedAppsListSchemaMock());
   public createEndpointList = jest.fn().mockResolvedValue(getExceptionListSchemaMock());
   public exportExceptionListAndItems = jest.fn().mockResolvedValue({
-    exportData: 'exportString',
+    exportData: `${JSON.stringify(getDetectionsExceptionListSchemaMock())}\n${JSON.stringify(
+      getExceptionListItemSchemaMock({ list_id: 'exception_list_id' })
+    )}`,
     exportDetails: {
-      exported_exception_list_count: 0,
-      exported_exception_list_item_count: 0,
+      exported_exception_list_count: 1,
+      exported_exception_list_item_count: 1,
       missing_exception_list_item_count: 0,
       missing_exception_list_items: [],
       missing_exception_lists: [],

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_all.test.ts

@@ -101,8 +101,8 @@ describe.each([
       exceptions_list: getListArrayMock(),
     });
     expect(detailsJson).toEqual({
-      exported_exception_list_count: 0,
-      exported_exception_list_item_count: 0,
+      exported_exception_list_count: 1,
+      exported_exception_list_item_count: 1,
       exported_rules_count: 1,
       missing_exception_list_item_count: 0,
       missing_exception_list_items: [],

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_rule_exceptions.test.ts

@@ -8,15 +8,15 @@
 import { ENDPOINT_LIST_ID } from '@kbn/securitysolution-list-constants';
 
 import { getExceptionListClientMock } from '../../../../../lists/server/services/exception_lists/exception_list_client.mock';
+import { getDetectionsExceptionListSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_schema.mock';
+import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
+
 import {
   getRuleExceptionsForExport,
   getExportableExceptions,
   getDefaultExportDetails,
 } from './get_export_rule_exceptions';
-import {
-  getListArrayMock,
-  getListMock,
-} from '../../../../common/detection_engine/schemas/types/lists.mock';
+import { getListMock } from '../../../../common/detection_engine/schemas/types/lists.mock';
 
 describe('get_export_rule_exceptions', () => {
   describe('getRuleExceptionsForExport', () => {
@@ -36,7 +36,24 @@ describe('get_export_rule_exceptions', () => {
         getExceptionListClientMock()
       );
 
-      expect(exportData).toEqual('exportString');
+      expect(exportData).toEqual(
+        `${JSON.stringify(getDetectionsExceptionListSchemaMock())}\n${JSON.stringify(
+          getExceptionListItemSchemaMock({ list_id: 'exception_list_id' })
+        )}`
+      );
+    });
+
+    test('it does not return duplicate exception lists', async () => {
+      const { exportData } = await getRuleExceptionsForExport(
+        [getListMock(), getListMock()],
+        getExceptionListClientMock()
+      );
+
+      expect(exportData).toEqual(
+        `${JSON.stringify(getDetectionsExceptionListSchemaMock())}\n${JSON.stringify(
+          getExceptionListItemSchemaMock({ list_id: 'exception_list_id' })
+        )}`
+      );
     });
 
     test('it does not return a global endpoint list', async () => {
@@ -60,11 +77,15 @@ describe('get_export_rule_exceptions', () => {
     test('it returns stringified exception lists and items', async () => {
       // This rule has 2 exception lists tied to it
       const { exportData } = await getExportableExceptions(
-        getListArrayMock(),
+        [getListMock()],
         getExceptionListClientMock()
       );
 
-      expect(exportData).toEqual('exportStringexportString');
+      expect(exportData).toEqual(
+        `${JSON.stringify(getDetectionsExceptionListSchemaMock())}\n${JSON.stringify(
+          getExceptionListItemSchemaMock({ list_id: 'exception_list_id' })
+        )}`
+      );
     });
 
     test('it throws error if error occurs in getting exceptions', async () => {
@@ -72,7 +93,7 @@ describe('get_export_rule_exceptions', () => {
       exceptionsClient.exportExceptionListAndItems = jest.fn().mockRejectedValue(new Error('oops'));
       // This rule has 2 exception lists tied to it
       await expect(async () => {
-        await getExportableExceptions(getListArrayMock(), exceptionsClient);
+        await getExportableExceptions([getListMock()], exceptionsClient);
       }).rejects.toThrowErrorMatchingInlineSnapshot(`"oops"`);
     });
   });

x-pack/plugins/security_solution/server/lib/detection_engine/rules/get_export_rule_exceptions.ts

@@ -21,10 +21,17 @@ export const getRuleExceptionsForExport = async (
   exceptions: ListArray,
   exceptionsListClient: ExceptionListClient | undefined
 ): Promise<ExportExceptionListAndItemsReturn> => {
+  const uniqueExceptionLists = new Set();
+
   if (exceptionsListClient != null) {
-    const exceptionsWithoutUnexportableLists = exceptions.filter(
-      ({ list_id: listId }) => !NON_EXPORTABLE_LIST_IDS.includes(listId)
-    );
+    const exceptionsWithoutUnexportableLists = exceptions.filter((list) => {
+      if (!uniqueExceptionLists.has(list.id)) {
+        uniqueExceptionLists.add(list.id);
+        return !NON_EXPORTABLE_LIST_IDS.includes(list.list_id);
+      } else {
+        return false;
+      }
+    });
     return getExportableExceptions(exceptionsWithoutUnexportableLists, exceptionsListClient);
   } else {
     return { exportData: '', exportDetails: getDefaultExportDetails() };
@@ -72,9 +79,9 @@ export const getExportableExceptions = async (
 };
 
 /**
- * Creates promises of the rules and returns them.
+ * Creates promises of the exceptions to be exported and returns them.
  * @param exceptionsListClient Exception Lists client
- * @param exceptions The rules to apply the update for
+ * @param exceptions The exceptions to be exported
  * @returns Promise of export ready exceptions.
  */
 export const createPromises = (

x-pack/plugins/security_solution/server/lib/detection_engine/scripts/check_env_variables.sh

@@ -30,13 +30,3 @@ if [ -z "${KIBANA_URL}" ]; then
   echo "Set KIBANA_URL in your environment"
   exit 1
 fi
-
-if [ -z "${TASK_MANAGER_INDEX}" ]; then
-  echo "Set TASK_MANAGER_INDEX in your environment"
-  exit 1
-fi
-
-if [ -z "${KIBANA_INDEX}" ]; then
-  echo "Set KIBANA_INDEX in your environment"
-  exit 1
-fi