Adds one time conflict retry and cleans up the exception lists to use the REST API (#115848)

## Summary

Improves FTR/e2e conflict retries with exception lists and security rules.

Fixes:
https://github.com/elastic/kibana/issues/115734
https://github.com/elastic/kibana/issues/115769
https://github.com/elastic/kibana/issues/115715
https://github.com/elastic/kibana/issues/115702
https://github.com/elastic/kibana/issues/115701

This past week we have been seeing increasing flake across tests involving `exception_lists` involving a `409 conflict` on our tests. Looking at each of the tests above and the flake it looks like we were calling Elasticsearch directly within the `.kibana` index to delete the exception list and list items as a shortcut:

```
export const deleteAllExceptions = async (es: KibanaClient): Promise<void> => {
  return countDownES(async () => {
    return es.deleteByQuery({
      index: '.kibana',
      q: 'type:exception-list or type:exception-list-agnostic',
      wait_for_completion: true,
      refresh: true,
      body: {},
    });
  }, 'deleteAllExceptions');
};
```

Although I think we did everything correctly `wait_for_completion: true` and  `refresh: true` within the tests there might be a slight race condition where the delete by query does not immediately happen for us. Since we should prefer to use direct REST API's where we can instead of calling into `.kibana` I changed this to using the exception list API:

```
export const deleteAllExceptions = async (
  supertest: SuperTest.SuperTest<SuperTest.Test>
): Promise<void> => {
  await countDownTest(
    async () => {
      const { body } = await supertest
        .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999`)
        .set('kbn-xsrf', 'true')
        .send();

      const ids: string[] = body.data.map((exception: ExceptionList) => exception.id);
      for await (const id of ids) {
        await supertest.delete(`${EXCEPTION_LIST_URL}?id=${id}`).set('kbn-xsrf', 'true').send();
      }
      const { body: finalCheck } = await supertest
        .get(`${EXCEPTION_LIST_URL}/_find`)
        .set('kbn-xsrf', 'true')
        .send();
      return finalCheck.data.length === 0;
    },
    'deleteAllExceptions',
    50,
    1000
  );
};
```

The additional final check above should ensure it sees that the data has been deleted before returning. Otherwise it will loop around again and keep trying.

I also improve both the `createRules` and `createExceptionList` by introducing a one-time, "detect if in conflict" and then "remove if in conflict" within those tests. This should help safe guard against flake if the above does not fix it. I also added more logging statements in case we do encounter this again on the CI system we can further trouble shoot it and add additional retry logic/fix logic.

A good side effect is if now you kill your tests half way through and restart them, the additional "detect if conflict" will recover your test for you as a developer. So 👍 that is an added benefit.

Example error message you would get (but not test failure) if you remove one of the cleanup sections in the `afterEach` or if you kill a test half way through and then restart it as an engineer:

```
└-: "is" operator
             └-> "before all" hook for "should find all the text from the data set when no exceptions are set on the rule"
             └-> should find all the text from the data set when no exceptions are set on the rule
               └-> "before each" hook: global before each for "should find all the text from the data set when no exceptions are set on the rule"
               └-> "before each" hook for "should find all the text from the data set when no exceptions are set on the rule"
When creating a rule found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: {"message":"rule_id: \"rule-1\" already exists","status_code":409}
               └- ✓ pass  (7.9s)
```

### Checklist

- [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_endpoint_exceptions.ts

@@ -69,7 +69,6 @@ export const getHostHits = async (
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for endpoints', () => {
     before(async () => {
@@ -94,7 +93,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_exceptions.ts

@@ -79,7 +79,7 @@ export default ({ getService }: FtrProviderContext) => {
       afterEach(async () => {
         await deleteSignalsIndex(supertest);
         await deleteAllAlerts(supertest);
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       describe('elastic admin', () => {
@@ -550,7 +550,7 @@ export default ({ getService }: FtrProviderContext) => {
         afterEach(async () => {
           await deleteSignalsIndex(supertest);
           await deleteAllAlerts(supertest);
-          await deleteAllExceptions(es);
+          await deleteAllExceptions(supertest);
         });
 
         it('should be able to execute against an exception list that does not include valid entries and get back 10 signals', async () => {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/create_ml.ts

@@ -211,9 +211,10 @@ export default ({ getService }: FtrProviderContext) => {
       const signalsOpen = await getOpenSignals(supertest, es, createdRule);
       expect(signalsOpen.hits.hits.length).eql(7);
     });
+
     describe('with non-value list exception', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
       it('generates no signals when an exception is added for an ML rule', async () => {
         const createdRule = await createRuleWithExceptionEntries(supertest, testRule, [
@@ -238,7 +239,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       afterEach(async () => {
         await deleteListsIndex(supertest);
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('generates no signals when a value list exception is added for an ML rule', async () => {

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/date.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type date', () => {
     before(async () => {
@@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/double.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type double', () => {
     before(async () => {
@@ -53,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/float.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type float', () => {
     before(async () => {
@@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/integer.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type integer', () => {
     before(async () => {
@@ -53,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type ip', () => {
     before(async () => {
@@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/ip_array.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type ip', () => {
     before(async () => {
@@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type keyword', () => {
     before(async () => {
@@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/keyword_array.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type keyword', () => {
     before(async () => {
@@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/long.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type long', () => {
     before(async () => {
@@ -51,7 +50,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text.ts

@@ -31,7 +31,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type text', () => {
     before(async () => {
@@ -52,7 +51,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/security_and_spaces/tests/exception_operators_data_types/text_array.ts

@@ -30,7 +30,6 @@ import {
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
   const esArchiver = getService('esArchiver');
-  const es = getService('es');
 
   describe('Rule exception operators for data type text', () => {
     before(async () => {
@@ -49,7 +48,7 @@ export default ({ getService }: FtrProviderContext) => {
     afterEach(async () => {
       await deleteSignalsIndex(supertest);
       await deleteAllAlerts(supertest);
-      await deleteAllExceptions(es);
+      await deleteAllExceptions(supertest);
       await deleteListsIndex(supertest);
     });
 

x-pack/test/detection_engine_api_integration/utils.ts

@@ -413,17 +413,12 @@ export const getSimpleMlRuleOutput = (ruleId = 'rule-1'): Partial<RulesSchema> =
  * @param supertest The supertest agent.
  */
 export const deleteAllAlerts = async (
-  supertest: SuperTest.SuperTest<SuperTest.Test>,
-  space?: string
+  supertest: SuperTest.SuperTest<SuperTest.Test>
 ): Promise<void> => {
   await countDownTest(
     async () => {
       const { body } = await supertest
-        .get(
-          space
-            ? `/s/${space}${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999`
-            : `${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999`
-        )
+        .get(`${DETECTION_ENGINE_RULES_URL}/_find?per_page=9999`)
         .set('kbn-xsrf', 'true')
         .send();
 
@@ -432,11 +427,7 @@ export const deleteAllAlerts = async (
       }));
 
       await supertest
-        .post(
-          space
-            ? `/s/${space}${DETECTION_ENGINE_RULES_URL}/_bulk_delete`
-            : `${DETECTION_ENGINE_RULES_URL}/_bulk_delete`
-        )
+        .post(`${DETECTION_ENGINE_RULES_URL}/_bulk_delete`)
         .send(ids)
         .set('kbn-xsrf', 'true');
 
@@ -899,8 +890,10 @@ export const countDownTest = async (
 };
 
 /**
- * Helper to cut down on the noise in some of the tests. This checks for
- * an expected 200 still and does not try to any retries.
+ * Helper to cut down on the noise in some of the tests. If this detects
+ * a conflict it will try to manually remove the rule before re-adding the rule one time and log
+ * and error about the race condition.
+ * rule a second attempt. It only re-tries adding the rule if it encounters a conflict once.
  * @param supertest The supertest deps
  * @param rule The rule to create
  */
@@ -908,17 +901,69 @@ export const createRule = async (
   supertest: SuperTest.SuperTest<SuperTest.Test>,
   rule: CreateRulesSchema
 ): Promise<FullResponseSchema> => {
-  const { body } = await supertest
+  const response = await supertest
     .post(DETECTION_ENGINE_RULES_URL)
     .set('kbn-xsrf', 'true')
-    .send(rule)
-    .expect(200);
-  return body;
+    .send(rule);
+  if (response.status === 409) {
+    if (rule.rule_id != null) {
+      // eslint-disable-next-line no-console
+      console.log(
+        `When creating a rule found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: ${JSON.stringify(
+          response.body
+        )}`
+      );
+      await deleteRule(supertest, rule.rule_id);
+      const secondResponseTry = await supertest
+        .post(DETECTION_ENGINE_RULES_URL)
+        .set('kbn-xsrf', 'true')
+        .send(rule);
+      if (secondResponseTry.status !== 200) {
+        throw new Error(
+          `Unexpected non 200 ok when attempting to create a rule (second try): ${JSON.stringify(
+            response.body
+          )}`
+        );
+      } else {
+        return secondResponseTry.body;
+      }
+    } else {
+      throw new Error('When creating a rule found an unexpected conflict (404)');
+    }
+  } else if (response.status !== 200) {
+    throw new Error(
+      `Unexpected non 200 ok when attempting to create a rule: ${JSON.stringify(response.status)}`
+    );
+  } else {
+    return response.body;
+  }
 };
 
 /**
- * Helper to cut down on the noise in some of the tests. This checks for
- * an expected 200 still and does not try to any retries.
+ * Helper to cut down on the noise in some of the tests. Does a delete of a rule.
+ * It does not check for a 200 "ok" on this.
+ * @param supertest The supertest deps
+ * @param id The rule id to delete
+ */
+export const deleteRule = async (
+  supertest: SuperTest.SuperTest<SuperTest.Test>,
+  ruleId: string
+): Promise<FullResponseSchema> => {
+  const response = await supertest
+    .delete(`${DETECTION_ENGINE_RULES_URL}?rule_id=${ruleId}`)
+    .set('kbn-xsrf', 'true');
+  if (response.status !== 200) {
+    // eslint-disable-next-line no-console
+    console.log(
+      'Did not get an expected 200 "ok" when deleting the rule. CI issues could happen. Suspect this line if you are seeing CI issues.'
+    );
+  }
+
+  return response.body;
+};
+
+/**
+ * Helper to cut down on the noise in some of the tests.
  * @param supertest The supertest deps
  * @param rule The rule to create
  */
@@ -1017,12 +1062,68 @@ export const createExceptionList = async (
   supertest: SuperTest.SuperTest<SuperTest.Test>,
   exceptionList: CreateExceptionListSchema
 ): Promise<ExceptionListSchema> => {
-  const { body } = await supertest
+  const response = await supertest
     .post(EXCEPTION_LIST_URL)
     .set('kbn-xsrf', 'true')
-    .send(exceptionList)
-    .expect(200);
-  return body;
+    .send(exceptionList);
+
+  if (response.status === 409) {
+    if (exceptionList.list_id != null) {
+      // eslint-disable-next-line no-console
+      console.log(
+        `When creating an exception list found an unexpected conflict (409), will attempt a cleanup and one time re-try. This usually indicates a bad cleanup or race condition within the tests: ${JSON.stringify(
+          response.body
+        )}`
+      );
+      await deleteExceptionList(supertest, exceptionList.list_id);
+      const secondResponseTry = await supertest
+        .post(EXCEPTION_LIST_URL)
+        .set('kbn-xsrf', 'true')
+        .send(exceptionList);
+      if (secondResponseTry.status !== 200) {
+        throw new Error(
+          `Unexpected non 200 ok when attempting to create an exception list (second try): ${JSON.stringify(
+            response.body
+          )}`
+        );
+      } else {
+        return secondResponseTry.body;
+      }
+    } else {
+      throw new Error('When creating an exception list found an unexpected conflict (404)');
+    }
+  } else if (response.status !== 200) {
+    throw new Error(
+      `Unexpected non 200 ok when attempting to create an exception list: ${JSON.stringify(
+        response.status
+      )}`
+    );
+  } else {
+    return response.body;
+  }
+};
+
+/**
+ * Helper to cut down on the noise in some of the tests. Does a delete of a rule.
+ * It does not check for a 200 "ok" on this.
+ * @param supertest The supertest deps
+ * @param id The rule id to delete
+ */
+export const deleteExceptionList = async (
+  supertest: SuperTest.SuperTest<SuperTest.Test>,
+  listId: string
+): Promise<FullResponseSchema> => {
+  const response = await supertest
+    .delete(`${EXCEPTION_LIST_URL}?list_id=${listId}`)
+    .set('kbn-xsrf', 'true');
+  if (response.status !== 200) {
+    // eslint-disable-next-line no-console
+    console.log(
+      'Did not get an expected 200 "ok" when deleting an exception list. CI issues could happen. Suspect this line if you are seeing CI issues.'
+    );
+  }
+
+  return response.body;
 };
 
 /**

x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_list_items.ts

@@ -27,7 +27,6 @@ import { deleteAllExceptions } from '../../utils';
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('create_exception_list_items', () => {
     describe('validation errors', () => {
@@ -47,7 +46,7 @@ export default ({ getService }: FtrProviderContext) => {
 
     describe('creating exception list items', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should create a simple exception list item with a list item id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/create_exception_lists.ts

@@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('create_exception_lists', () => {
     describe('creating exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should create a simple exception list', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_list_items.ts

@@ -22,12 +22,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties }
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('delete_exception_list_items', () => {
     describe('delete exception list items', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should delete a single exception list item by its item_id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/delete_exception_lists.ts

@@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('delete_exception_lists', () => {
     describe('delete exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should delete a single exception list by its list_id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/delete_lists.ts

@@ -33,7 +33,6 @@ import { DETECTION_TYPE, LIST_ID } from '../../../../plugins/lists/common/consta
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('delete_lists', () => {
     describe('deleting lists', () => {
@@ -117,7 +116,7 @@ export default ({ getService }: FtrProviderContext) => {
 
       describe('deleting lists referenced in exceptions', () => {
         afterEach(async () => {
-          await deleteAllExceptions(es);
+          await deleteAllExceptions(supertest);
         });
 
         it('should return an error when deleting a list referenced within an exception list item', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/export_exception_list.ts

@@ -22,12 +22,11 @@ import { getCreateExceptionListItemMinimalSchemaMock } from '../../../../plugins
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('export_exception_list_route', () => {
     describe('exporting exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should set the response content types to be expected', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_list_items.ts

@@ -18,12 +18,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties }
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('find_exception_list_items', () => {
     describe('find exception list items', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should return an empty find body correctly if no exception list items are loaded', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/find_exception_lists.ts

@@ -17,12 +17,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext): void => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('find_exception_lists', () => {
     describe('find exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should return an empty find body correctly if no exception lists are loaded', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_list_items.ts

@@ -22,12 +22,11 @@ import { deleteAllExceptions, removeExceptionListItemServerGeneratedProperties }
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('read_exception_list_items', () => {
     describe('reading exception list items', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should be able to read a single exception list items using item_id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/read_exception_lists.ts

@@ -21,12 +21,11 @@ import { deleteAllExceptions, removeExceptionListServerGeneratedProperties } fro
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('read_exception_lists', () => {
     describe('reading exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should be able to read a single exception list using list_id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/summary_exception_lists.ts

@@ -21,7 +21,6 @@ interface SummaryResponseType {
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('summary_exception_lists', () => {
     describe('summary exception lists', () => {
@@ -30,7 +29,7 @@ export default ({ getService }: FtrProviderContext) => {
       });
       afterEach(async () => {
         await deleteListsIndex(supertest);
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should give a validation error if the list_id and the id are not supplied', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_list_items.ts

@@ -24,12 +24,11 @@ import { getUpdateMinimalExceptionListItemSchemaMock } from '../../../../plugins
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('update_exception_list_items', () => {
     describe('update exception list items', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should update a single exception list item property of name using an id', async () => {

x-pack/test/lists_api_integration/security_and_spaces/tests/update_exception_lists.ts

@@ -23,12 +23,11 @@ import { getUpdateMinimalExceptionListSchemaMock } from '../../../../plugins/lis
 // eslint-disable-next-line import/no-default-export
 export default ({ getService }: FtrProviderContext) => {
   const supertest = getService('supertest');
-  const es = getService('es');
 
   describe('update_exception_lists', () => {
     describe('update exception lists', () => {
       afterEach(async () => {
-        await deleteAllExceptions(es);
+        await deleteAllExceptions(supertest);
       });
 
       it('should update a single exception list property of name using an id', async () => {

x-pack/test/lists_api_integration/utils.ts

@@ -6,7 +6,6 @@
  */
 
 import type SuperTest from 'supertest';
-import type { KibanaClient } from '@elastic/elasticsearch/api/kibana';
 
 import type {
   Type,
@@ -14,10 +13,15 @@ import type {
   ListItemSchema,
   ExceptionListSchema,
   ExceptionListItemSchema,
+  ExceptionList,
 } from '@kbn/securitysolution-io-ts-list-types';
-import { LIST_INDEX, LIST_ITEM_URL } from '@kbn/securitysolution-list-constants';
+import {
+  EXCEPTION_LIST_URL,
+  LIST_INDEX,
+  LIST_ITEM_URL,
+} from '@kbn/securitysolution-list-constants';
 import { getImportListItemAsBuffer } from '../../plugins/lists/common/schemas/request/import_list_item_schema.mock';
-import { countDownES, countDownTest } from '../detection_engine_api_integration/utils';
+import { countDownTest } from '../detection_engine_api_integration/utils';
 
 /**
  * Creates the lists and lists items index for use inside of beforeEach blocks of tests
@@ -160,20 +164,34 @@ export const binaryToString = (res: any, callback: any): void => {
 };
 
 /**
- * Remove all exceptions from the .kibana index
- * This will retry 20 times before giving up and hopefully still not interfere with other tests
- * @param es The ElasticSearch handle
+ * Remove all exceptions
+ * This will retry 50 times before giving up and hopefully still not interfere with other tests
+ * @param supertest The supertest handle
  */
-export const deleteAllExceptions = async (es: KibanaClient): Promise<void> => {
-  return countDownES(async () => {
-    return es.deleteByQuery({
-      index: '.kibana',
-      q: 'type:exception-list or type:exception-list-agnostic',
-      wait_for_completion: true,
-      refresh: true,
-      body: {},
-    });
-  }, 'deleteAllExceptions');
+export const deleteAllExceptions = async (
+  supertest: SuperTest.SuperTest<SuperTest.Test>
+): Promise<void> => {
+  await countDownTest(
+    async () => {
+      const { body } = await supertest
+        .get(`${EXCEPTION_LIST_URL}/_find?per_page=9999`)
+        .set('kbn-xsrf', 'true')
+        .send();
+
+      const ids: string[] = body.data.map((exception: ExceptionList) => exception.id);
+      for await (const id of ids) {
+        await supertest.delete(`${EXCEPTION_LIST_URL}?id=${id}`).set('kbn-xsrf', 'true').send();
+      }
+      const { body: finalCheck } = await supertest
+        .get(`${EXCEPTION_LIST_URL}/_find`)
+        .set('kbn-xsrf', 'true')
+        .send();
+      return finalCheck.data.length === 0;
+    },
+    'deleteAllExceptions',
+    50,
+    1000
+  );
 };
 
 /**