From d381d486454798419de7177915002d74eca0a337 Mon Sep 17 00:00:00 2001 From: Rudolf Meijering Date: Tue, 23 Feb 2021 13:54:57 +0100 Subject: [PATCH] v1 migrations: drop fleet-agent-events during a migration (#92188) (#92371) * v1 migrations: drop fleet-agent-events during a migration * Add TODO to fleet to make it clear that fleet-agent-events should not be used * Fix test --- .../migrations/core/elastic_index.test.ts | 13 ++- .../migrations/core/elastic_index.ts | 19 ++++- .../apis/saved_objects/migrations.ts | 80 +++++++++++++++++++ .../plugins/fleet/common/constants/agent.ts | 2 + .../fleet/server/saved_objects/index.ts | 2 + 5 files changed, 114 insertions(+), 2 deletions(-) diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts index b493501e09c6..bfa686ac0cc4 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts @@ -432,7 +432,18 @@ describe('ElasticIndex', () => { expect(await read()).toEqual([]); expect(client.search).toHaveBeenCalledWith({ - body: { size: 100 }, + body: { + size: 100, + query: { + bool: { + must_not: { + term: { + type: 'fleet-agent-events', + }, + }, + }, + }, + }, index, scroll: '5m', }); diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts index cb1120eb2ff4..e42643565eb4 100644 --- a/src/core/server/saved_objects/migrations/core/elastic_index.ts +++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts @@ -67,6 +67,20 @@ export function reader( const scroll = scrollDuration; let scrollId: string | undefined; + // When migrating from the outdated index we use a read query which excludes + // saved objects which are no longer used. These saved objects will still be + // kept in the outdated index for backup purposes, but won't be availble in + // the upgraded index. + const excludeUnusedTypes = { + bool: { + must_not: { + term: { + type: 'fleet-agent-events', // https://github.com/elastic/kibana/issues/91869 + }, + }, + }, + }; + const nextBatch = () => scrollId !== undefined ? client.scroll>({ @@ -74,7 +88,10 @@ export function reader( scroll_id: scrollId, }) : client.search>({ - body: { size: batchSize }, + body: { + size: batchSize, + query: excludeUnusedTypes, + }, index, scroll, }); diff --git a/test/api_integration/apis/saved_objects/migrations.ts b/test/api_integration/apis/saved_objects/migrations.ts index 5a5158825a22..1f1f1a5c98cd 100644 --- a/test/api_integration/apis/saved_objects/migrations.ts +++ b/test/api_integration/apis/saved_objects/migrations.ts @@ -48,6 +48,12 @@ const BAZ_TYPE: SavedObjectsType = { namespaceType: 'single', mappings: { properties: {} }, }; +const FLEET_AGENT_EVENT_TYPE: SavedObjectsType = { + name: 'fleet-agent-event', + hidden: false, + namespaceType: 'single', + mappings: { properties: {} }, +}; function getLogMock() { return { @@ -331,6 +337,80 @@ export default ({ getService }: FtrProviderContext) => { ]); }); + it('drops fleet-agent-event saved object types when doing a migration', async () => { + const index = '.migration-b'; + const originalDocs = [ + { + id: 'fleet-agent-event:a', + type: 'fleet-agent-event', + 'fleet-agent-event': { name: 'Foo A' }, + }, + { + id: 'fleet-agent-event:e', + type: 'fleet-agent-event', + 'fleet-agent-event': { name: 'Fooey' }, + }, + { id: 'bar:i', type: 'bar', bar: { nomnom: 33 } }, + { id: 'bar:o', type: 'bar', bar: { nomnom: 2 } }, + ]; + + const mappingProperties = { + 'fleet-agent-event': { properties: { name: { type: 'text' } } }, + bar: { properties: { mynum: { type: 'integer' } } }, + }; + + let savedObjectTypes: SavedObjectsType[] = [ + FLEET_AGENT_EVENT_TYPE, + { + ...BAR_TYPE, + migrations: { + '1.0.0': (doc) => set(doc, 'attributes.nomnom', doc.attributes.nomnom + 1), + '1.3.0': (doc) => set(doc, 'attributes', { mynum: doc.attributes.nomnom }), + '1.9.0': (doc) => set(doc, 'attributes.mynum', doc.attributes.mynum * 2), + }, + }, + ]; + + await createIndex({ esClient, index, esDeleteAllIndices }); + await createDocs({ esClient, index, docs: originalDocs }); + + await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); + + // @ts-expect-error name doesn't exist on mynum type + mappingProperties.bar.properties.name = { type: 'keyword' }; + savedObjectTypes = [ + FLEET_AGENT_EVENT_TYPE, + { + ...BAR_TYPE, + migrations: { + '2.3.4': (doc) => set(doc, 'attributes.name', `NAME ${doc.id}`), + }, + }, + ]; + + await migrateIndex({ esClient, index, savedObjectTypes, mappingProperties }); + + // Assert that fleet-agent-events were dropped + expect(await fetchDocs(esClient, index)).to.eql([ + { + id: 'bar:i', + type: 'bar', + migrationVersion: { bar: '2.3.4' }, + bar: { mynum: 68, name: 'NAME i' }, + references: [], + coreMigrationVersion: KIBANA_VERSION, + }, + { + id: 'bar:o', + type: 'bar', + migrationVersion: { bar: '2.3.4' }, + bar: { mynum: 6, name: 'NAME o' }, + references: [], + coreMigrationVersion: KIBANA_VERSION, + }, + ]); + }); + it('Coordinates migrations across the Kibana cluster', async () => { const index = '.migration-c'; const originalDocs = [{ id: 'foo:lotr', type: 'foo', foo: { name: 'Lord of the Rings' } }]; diff --git a/x-pack/plugins/fleet/common/constants/agent.ts b/x-pack/plugins/fleet/common/constants/agent.ts index d3467973fad9..92e24256c7a2 100644 --- a/x-pack/plugins/fleet/common/constants/agent.ts +++ b/x-pack/plugins/fleet/common/constants/agent.ts @@ -6,6 +6,8 @@ */ export const AGENT_SAVED_OBJECT_TYPE = 'fleet-agents'; +// TODO: Remove this saved object type. Core will drop any saved objects of +// this type during migrations. See https://github.com/elastic/kibana/issues/91869 export const AGENT_EVENT_SAVED_OBJECT_TYPE = 'fleet-agent-events'; export const AGENT_ACTION_SAVED_OBJECT_TYPE = 'fleet-agent-actions'; diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts index 5b851c692ad3..d6e3cba99890 100644 --- a/x-pack/plugins/fleet/server/saved_objects/index.ts +++ b/x-pack/plugins/fleet/server/saved_objects/index.ts @@ -124,6 +124,8 @@ const getSavedObjectTypes = ( '7.10.0': migrateAgentActionToV7100(encryptedSavedObjects), }, }, + // TODO: Remove this saved object type. Core will drop any saved objects of + // this type during migrations. See https://github.com/elastic/kibana/issues/91869 [AGENT_EVENT_SAVED_OBJECT_TYPE]: { name: AGENT_EVENT_SAVED_OBJECT_TYPE, hidden: false,