[ILM] Surface linked hidden indices for policies (#87960)

x-pack/plugins/index_lifecycle_management/server/routes/api/policies/register_fetch_route.ts

@@ -43,7 +43,7 @@ async function addLinkedIndices(client: ElasticsearchClient, policiesMap: Polici
 
   const response = await client.ilm.explainLifecycle<{
     indices: { [indexName: string]: IndexLifecyclePolicy };
-  }>({ index: '*' }, options);
+  }>({ index: '*,.*' }, options); // '*,.*' will include hidden indices
   const policyExplanation = response.body;
   Object.entries(policyExplanation.indices).forEach(([indexName, { policy }]) => {
     if (policy && policiesMap[policy]) {

x-pack/test/api_integration/apis/management/index_lifecycle_management/lib/elasticsearch.js

@@ -13,6 +13,8 @@ import { getRandomString } from './random';
 export const initElasticsearchHelpers = (es) => {
   let indicesCreated = [];
   let templatesCreated = [];
+  let composableTemplatesCreated = [];
+  let dataStreamsCreated = [];
 
   // Indices
   const getIndex = (index) => es.indices.get({ index }).then(({ body }) => body);
@@ -30,6 +32,20 @@ export const initElasticsearchHelpers = (es) => {
   const deleteAllIndices = () =>
     Promise.all(indicesCreated.map(deleteIndex)).then(() => (indicesCreated = []));
 
+  // Data streams
+  const createDataStream = (dataStream = getRandomString(), document) => {
+    dataStreamsCreated.push(dataStream);
+    return es.index({ index: dataStream, body: document });
+  };
+
+  const deleteDataStream = (dataStream) => {
+    dataStreamsCreated = dataStreamsCreated.filter((i) => i !== dataStream);
+    return es.indices.deleteDataStream({ name: dataStream });
+  };
+
+  const deleteAllDataStreams = () =>
+    Promise.all(dataStreamsCreated.map(deleteDataStream)).then(() => (dataStreamsCreated = []));
+
   // Index templates
   const getIndexTemplates = () => es.indices.getTemplate();
 
@@ -39,6 +55,11 @@ export const initElasticsearchHelpers = (es) => {
     return es.indices.putTemplate({ name, body: template }, { create: true });
   };
 
+  const createComposableIndexTemplate = (name, template) => {
+    composableTemplatesCreated.push(name);
+    return es.indices.putIndexTemplate({ name, body: template }, { create: true });
+  };
+
   const deleteIndexTemplate = (name) => {
     templatesCreated = templatesCreated.filter((i) => i !== name);
     return es.indices.deleteTemplate({ name }).catch((err) => {
@@ -49,22 +70,45 @@ export const initElasticsearchHelpers = (es) => {
     });
   };
 
+  const deleteComposableIndexTemplate = (name) => {
+    composableTemplatesCreated = composableTemplatesCreated.filter((i) => i !== name);
+    return es.indices.deleteIndexTemplate({ name }).catch((err) => {
+      // Silently fail if templates not found
+      if (err.statusCode !== 404) {
+        throw err;
+      }
+    });
+  };
+
   const deleteAllTemplates = () =>
     Promise.all(templatesCreated.map(deleteIndexTemplate)).then(() => (templatesCreated = []));
 
-  const cleanUp = () => Promise.all([deleteAllIndices(), deleteAllTemplates()]);
+  const deleteAllComposableTemplates = () =>
+    Promise.all(templatesCreated.map(deleteComposableIndexTemplate)).then(
+      () => (composableTemplatesCreated = [])
+    );
+
+  const cleanUp = () =>
+    Promise.all([
+      deleteAllIndices(),
+      deleteAllTemplates(),
+      deleteAllComposableTemplates(),
+      deleteAllDataStreams(),
+    ]);
 
   const getNodesStats = () => es.nodes.stats().then(({ body }) => body);
 
   return {
     getIndex,
     createIndex,
+    createDataStream,
     deleteIndex,
     deleteAllIndices,
     deleteAllTemplates,
     getIndexTemplates,
     createIndexTemplate,
     deleteIndexTemplate,
+    createComposableIndexTemplate,
     getNodesStats,
     cleanUp,
   };

x-pack/test/api_integration/apis/management/index_lifecycle_management/policies.js

@@ -17,7 +17,12 @@ export default function ({ getService }) {
 
   const es = getService('es');
 
-  const { createIndex, cleanUp: cleanUpEsResources } = initElasticsearchHelpers(es);
+  const {
+    createIndex,
+    createComposableIndexTemplate,
+    createDataStream,
+    cleanUp: cleanUpEsResources,
+  } = initElasticsearchHelpers(es);
 
   const {
     loadPolicies,
@@ -74,6 +79,34 @@ export default function ({ getService }) {
         const fetchedPolicy = body.find((p) => p.name === policyName);
         expect(fetchedPolicy.linkedIndices).to.eql([indexName]);
       });
+
+      it('should add hidden indices linked to policies', async () => {
+        // Create a policy
+        const policy = getPolicyPayload('hidden-index-link-test-policy');
+        const { name: policyName } = policy;
+        await createPolicy(policy);
+
+        // Create hidden data stream
+        await createComposableIndexTemplate('my_template', {
+          template: {},
+          index_patterns: ['hidden*'],
+          data_stream: {
+            hidden: true,
+          },
+        });
+
+        const indexName = 'hidden_index';
+        await createDataStream(indexName, {
+          '@timestamp': '2020-01-27',
+        });
+
+        await addPolicyToIndex(policyName, indexName);
+
+        const { body } = await loadPolicies(true);
+        const fetchedPolicy = body.find((p) => p.name === policyName);
+        // The index name is dynamically generated as .ds-<indexName>-XXX so we don't check for exact match
+        expect(fetchedPolicy.linkedIndices[0]).to.contain(indexName);
+      });
     });
 
     describe('create', () => {