v1 migrations: drop fleet-agent-events during a migration (#92188) (#92371)

* v1 migrations: drop fleet-agent-events during a migration

* Add TODO to fleet to make it clear that fleet-agent-events should not be used

* Fix test

src/core/server/saved_objects/migrations/core/elastic_index.test.ts

@@ -432,7 +432,18 @@ describe('ElasticIndex', () => {
       expect(await read()).toEqual([]);
 
       expect(client.search).toHaveBeenCalledWith({
-        body: { size: 100 },
+        body: {
+          size: 100,
+          query: {
+            bool: {
+              must_not: {
+                term: {
+                  type: 'fleet-agent-events',
+                },
+              },
+            },
+          },
+        },
         index,
         scroll: '5m',
       });

src/core/server/saved_objects/migrations/core/elastic_index.ts

@@ -67,6 +67,20 @@ export function reader(
   const scroll = scrollDuration;
   let scrollId: string | undefined;
 
+  // When migrating from the outdated index we use a read query which excludes
+  // saved objects which are no longer used. These saved objects will still be
+  // kept in the outdated index for backup purposes, but won't be availble in
+  // the upgraded index.
+  const excludeUnusedTypes = {
+    bool: {
+      must_not: {
+        term: {
+          type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869
+        },
+      },
+    },
+  };
+
   const nextBatch = () =>
     scrollId !== undefined
       ? client.scroll<SearchResponse<SavedObjectsRawDocSource>>({
@@ -74,7 +88,10 @@ export function reader(
           scroll_id: scrollId,
         })
       : client.search<SearchResponse<SavedObjectsRawDocSource>>({
-          body: { size: batchSize },
+          body: {
+            size: batchSize,
+            query: excludeUnusedTypes,
+          },
           index,
           scroll,
         });

test/api_integration/apis/saved_objects/migrations.ts

@@ -48,6 +48,12 @@ const BAZ_TYPE: SavedObjectsType = {
   namespaceType: 'single',
   mappings: { properties: {} },
 };
+const FLEET_AGENT_EVENT_TYPE: SavedObjectsType = {
+  name: 'fleet-agent-event',
+  hidden: false,
+  namespaceType: 'single',
+  mappings: { properties: {} },
+};
 
 function getLogMock() {
   return {
@@ -331,6 +337,80 @@ export default ({ getService }: FtrProviderContext) => {
       ]);
     });
 
+    it('drops fleet-agent-event saved object types when doing a migration', async () => {
+      const index = '.migration-b';
+      const originalDocs = [
+        {
+          id: 'fleet-agent-event:a',
+          type: 'fleet-agent-event',
+          'fleet-agent-event': { name: 'Foo A' },
+        },
+        {
+          id: 'fleet-agent-event:e',
+          type: 'fleet-agent-event',
+          'fleet-agent-event': { name: 'Fooey' },
+        },
+        { id: 'bar:i', type: 'bar', bar: { nomnom: 33 } },
+        { id: 'bar:o', type: 'bar', bar: { nomnom: 2 } },
+      ];
+
+      const mappingProperties = {
+        'fleet-agent-event': { properties: { name: { type: 'text' } } },
+        bar: { properties: { mynum: { type: 'integer' } } },
+      };
+
+      let savedObjectTypes: SavedObjectsType[] = [
+        FLEET_AGENT_EVENT_TYPE,
+        {
+          ...BAR_TYPE,
+          migrations: {
+            '1.0.0': (doc) => set(doc, 'attributes.nomnom', doc.attributes.nomnom + 1),
+            '1.3.0': (doc) => set(doc, 'attributes', { mynum: doc.attributes.nomnom }),
+            '1.9.0': (doc) => set(doc, 'attributes.mynum', doc.attributes.mynum * 2),
+          },
+        },
+      ];
+
+      await createIndex({ esClient, index, esDeleteAllIndices });
+      await createDocs({ esClient, index, docs: originalDocs });
+
+      await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties });
+
+      // @ts-expect-error name doesn't exist on mynum type
+      mappingProperties.bar.properties.name = { type: 'keyword' };
+      savedObjectTypes = [
+        FLEET_AGENT_EVENT_TYPE,
+        {
+          ...BAR_TYPE,
+          migrations: {
+            '2.3.4': (doc) => set(doc, 'attributes.name', `NAME ${doc.id}`),
+          },
+        },
+      ];
+
+      await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties });
+
+      // Assert that fleet-agent-events were dropped
+      expect(await fetchDocs(esClient, index)).to.eql([
+        {
+          id: 'bar:i',
+          type: 'bar',
+          migrationVersion: { bar: '2.3.4' },
+          bar: { mynum: 68, name: 'NAME i' },
+          references: [],
+          coreMigrationVersion: KIBANA_VERSION,
+        },
+        {
+          id: 'bar:o',
+          type: 'bar',
+          migrationVersion: { bar: '2.3.4' },
+          bar: { mynum: 6, name: 'NAME o' },
+          references: [],
+          coreMigrationVersion: KIBANA_VERSION,
+        },
+      ]);
+    });
+
     it('Coordinates migrations across the Kibana cluster', async () => {
       const index = '.migration-c';
       const originalDocs = [{ id: 'foo:lotr', type: 'foo', foo: { name: 'Lord of the Rings' } }];

x-pack/plugins/fleet/common/constants/agent.ts

@@ -6,6 +6,8 @@
  */
 
 export const AGENT_SAVED_OBJECT_TYPE = 'fleet-agents';
+// TODO: Remove this saved object type. Core will drop any saved objects of
+// this type during migrations. See https://github.com/elastic/kibana/issues/91869
 export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'fleet-agent-events';
 export const AGENT_ACTION_SAVED_OBJECT_TYPE = 'fleet-agent-actions';
 

x-pack/plugins/fleet/server/saved_objects/index.ts

@@ -124,6 +124,8 @@ const getSavedObjectTypes = (
       '7.10.0': migrateAgentActionToV7100(encryptedSavedObjects),
     },
   },
+  // TODO: Remove this saved object type. Core will drop any saved objects of
+  // this type during migrations. See https://github.com/elastic/kibana/issues/91869
   [AGENT_EVENT_SAVED_OBJECT_TYPE]: {
     name: AGENT_EVENT_SAVED_OBJECT_TYPE,
     hidden: false,