From d9bd7b4fd666efc1ce4dc5a0e9e767f640783693 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Wed, 13 Jan 2021 14:50:01 -0500 Subject: [PATCH] [ILM] Surface linked hidden indices for policies (#87960) --- .../api/policies/register_fetch_route.ts | 2 +- .../lib/elasticsearch.js | 46 ++++++++++++++++++- .../index_lifecycle_management/policies.js | 35 +++++++++++++- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_fetch_route.ts b/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_fetch_route.ts index fc5f369e588f..1cf84c70ff90 100644 --- a/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_fetch_route.ts +++ b/x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_fetch_route.ts @@ -43,7 +43,7 @@ async function addLinkedIndices(client: ElasticsearchClient, policiesMap: Polici const response = await client.ilm.explainLifecycle<{ indices: { [indexName: string]: IndexLifecyclePolicy }; - }>({ index: '*' }, options); + }>({ index: '*,.*' }, options); // '*,.*' will include hidden indices const policyExplanation = response.body; Object.entries(policyExplanation.indices).forEach(([indexName, { policy }]) => { if (policy && policiesMap[policy]) { diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/lib/elasticsearch.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/lib/elasticsearch.js index 358e54d8738f..cf43ebf01b61 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/lib/elasticsearch.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/lib/elasticsearch.js @@ -13,6 +13,8 @@ import { getRandomString } from './random'; export const initElasticsearchHelpers = (es) => { let indicesCreated = []; let templatesCreated = []; + let composableTemplatesCreated = []; + let dataStreamsCreated = []; // Indices const getIndex = (index) => es.indices.get({ index }).then(({ body }) => body); @@ -30,6 +32,20 @@ export const initElasticsearchHelpers = (es) => { const deleteAllIndices = () => Promise.all(indicesCreated.map(deleteIndex)).then(() => (indicesCreated = [])); + // Data streams + const createDataStream = (dataStream = getRandomString(), document) => { + dataStreamsCreated.push(dataStream); + return es.index({ index: dataStream, body: document }); + }; + + const deleteDataStream = (dataStream) => { + dataStreamsCreated = dataStreamsCreated.filter((i) => i !== dataStream); + return es.indices.deleteDataStream({ name: dataStream }); + }; + + const deleteAllDataStreams = () => + Promise.all(dataStreamsCreated.map(deleteDataStream)).then(() => (dataStreamsCreated = [])); + // Index templates const getIndexTemplates = () => es.indices.getTemplate(); @@ -39,6 +55,11 @@ export const initElasticsearchHelpers = (es) => { return es.indices.putTemplate({ name, body: template }, { create: true }); }; + const createComposableIndexTemplate = (name, template) => { + composableTemplatesCreated.push(name); + return es.indices.putIndexTemplate({ name, body: template }, { create: true }); + }; + const deleteIndexTemplate = (name) => { templatesCreated = templatesCreated.filter((i) => i !== name); return es.indices.deleteTemplate({ name }).catch((err) => { @@ -49,22 +70,45 @@ export const initElasticsearchHelpers = (es) => { }); }; + const deleteComposableIndexTemplate = (name) => { + composableTemplatesCreated = composableTemplatesCreated.filter((i) => i !== name); + return es.indices.deleteIndexTemplate({ name }).catch((err) => { + // Silently fail if templates not found + if (err.statusCode !== 404) { + throw err; + } + }); + }; + const deleteAllTemplates = () => Promise.all(templatesCreated.map(deleteIndexTemplate)).then(() => (templatesCreated = [])); - const cleanUp = () => Promise.all([deleteAllIndices(), deleteAllTemplates()]); + const deleteAllComposableTemplates = () => + Promise.all(templatesCreated.map(deleteComposableIndexTemplate)).then( + () => (composableTemplatesCreated = []) + ); + + const cleanUp = () => + Promise.all([ + deleteAllIndices(), + deleteAllTemplates(), + deleteAllComposableTemplates(), + deleteAllDataStreams(), + ]); const getNodesStats = () => es.nodes.stats().then(({ body }) => body); return { getIndex, createIndex, + createDataStream, deleteIndex, deleteAllIndices, deleteAllTemplates, getIndexTemplates, createIndexTemplate, deleteIndexTemplate, + createComposableIndexTemplate, getNodesStats, cleanUp, }; diff --git a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js index 1589baabb1de..52979ed208bd 100644 --- a/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js +++ b/x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js @@ -17,7 +17,12 @@ export default function ({ getService }) { const es = getService('es'); - const { createIndex, cleanUp: cleanUpEsResources } = initElasticsearchHelpers(es); + const { + createIndex, + createComposableIndexTemplate, + createDataStream, + cleanUp: cleanUpEsResources, + } = initElasticsearchHelpers(es); const { loadPolicies, @@ -74,6 +79,34 @@ export default function ({ getService }) { const fetchedPolicy = body.find((p) => p.name === policyName); expect(fetchedPolicy.linkedIndices).to.eql([indexName]); }); + + it('should add hidden indices linked to policies', async () => { + // Create a policy + const policy = getPolicyPayload('hidden-index-link-test-policy'); + const { name: policyName } = policy; + await createPolicy(policy); + + // Create hidden data stream + await createComposableIndexTemplate('my_template', { + template: {}, + index_patterns: ['hidden*'], + data_stream: { + hidden: true, + }, + }); + + const indexName = 'hidden_index'; + await createDataStream(indexName, { + '@timestamp': '2020-01-27', + }); + + await addPolicyToIndex(policyName, indexName); + + const { body } = await loadPolicies(true); + const fetchedPolicy = body.find((p) => p.name === policyName); + // The index name is dynamically generated as .ds--XXX so we don't check for exact match + expect(fetchedPolicy.linkedIndices[0]).to.contain(indexName); + }); }); describe('create', () => {