[Upgrade Assistant] Hide system indices from es deprecations list (#113627)

x-pack/plugins/upgrade_assistant/common/types.ts

@@ -260,3 +260,19 @@ export interface DeprecationLoggingStatus {
   isDeprecationLogIndexingEnabled: boolean;
   isDeprecationLoggingEnabled: boolean;
 }
+
+export type UPGRADE_STATUS = 'UPGRADE_NEEDED' | 'NO_UPGRADE_NEEDED' | 'IN_PROGRESS';
+export interface SystemIndicesUpgradeFeature {
+  id?: string;
+  feature_name: string;
+  minimum_index_version: string;
+  upgrade_status: UPGRADE_STATUS;
+  indices: Array<{
+    index: string;
+    version: string;
+  }>;
+}
+export interface SystemIndicesUpgradeStatus {
+  features: SystemIndicesUpgradeFeature[];
+  upgrade_status: UPGRADE_STATUS;
+}

x-pack/plugins/upgrade_assistant/server/lib/__fixtures__/fake_deprecations.json

@@ -102,6 +102,15 @@
         "resolve_during_rolling_upgrade": false
       }
     ],
+    ".ml-config": [
+      {
+        "level": "critical",
+        "message": "Index created before 7.0",
+        "url": "https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_fields",
+        "details": "This index was created using version: 6.8.16",
+        "resolve_during_rolling_upgrade": false
+      }
+    ],
     ".watcher-history-6-2018.11.07": [
       {
         "level": "warning",

x-pack/plugins/upgrade_assistant/server/lib/es_deprecations_status.test.ts

@@ -40,6 +40,25 @@ describe('getESUpgradeStatus', () => {
     asApiResponse(deprecationsResponse)
   );
 
+  esClient.asCurrentUser.transport.request.mockResolvedValue(
+    asApiResponse({
+      features: [
+        {
+          feature_name: 'machine_learning',
+          minimum_index_version: '7.1.1',
+          upgrade_status: 'UPGRADE_NEEDED',
+          indices: [
+            {
+              index: '.ml-config',
+              version: '7.1.1',
+            },
+          ],
+        },
+      ],
+      upgrade_status: 'UPGRADE_NEEDED',
+    })
+  );
+
   // @ts-expect-error not full interface of response
   esClient.asCurrentUser.indices.resolveIndex.mockResolvedValue(asApiResponse(resolvedIndices));
 
@@ -86,4 +105,30 @@ describe('getESUpgradeStatus', () => {
       0
     );
   });
+
+  it('filters out system indices returned by upgrade system indices API', async () => {
+    esClient.asCurrentUser.migration.deprecations.mockResolvedValue(
+      asApiResponse({
+        cluster_settings: [],
+        node_settings: [],
+        ml_settings: [],
+        index_settings: {
+          '.ml-config': [
+            {
+              level: 'critical',
+              message: 'Index created before 7.0',
+              url: 'https://',
+              details: '...',
+              resolve_during_rolling_upgrade: false,
+            },
+          ],
+        },
+      })
+    );
+
+    const upgradeStatus = await getESUpgradeStatus(esClient);
+
+    expect(upgradeStatus.deprecations).toHaveLength(0);
+    expect(upgradeStatus.totalCriticalDeprecations).toBe(0);
+  });
 });

x-pack/plugins/upgrade_assistant/server/lib/es_deprecations_status.ts

@@ -14,6 +14,10 @@ import { indexSettingDeprecations } from '../../common/constants';
 import { EnrichedDeprecationInfo, ESUpgradeStatus } from '../../common/types';
 
 import { esIndicesStateCheck } from './es_indices_state_check';
+import {
+  getESSystemIndicesUpgradeStatus,
+  convertFeaturesToIndicesArray,
+} from '../lib/es_system_indices_upgrade';
 
 export async function getESUpgradeStatus(
   dataClient: IScopedClusterClient
@@ -22,10 +26,19 @@ export async function getESUpgradeStatus(
 
   const getCombinedDeprecations = async () => {
     const indices = await getCombinedIndexInfos(deprecations, dataClient);
+    const systemIndices = await getESSystemIndicesUpgradeStatus(dataClient.asCurrentUser);
+    const systemIndicesList = convertFeaturesToIndicesArray(systemIndices.features);
 
     return Object.keys(deprecations).reduce((combinedDeprecations, deprecationType) => {
       if (deprecationType === 'index_settings') {
-        combinedDeprecations = combinedDeprecations.concat(indices);
+        // We need to exclude all index related deprecations for system indices since
+        // they are resolved separately through the system indices upgrade section in
+        // the Overview page.
+        const withoutSystemIndices = indices.filter(
+          (index) => !systemIndicesList.includes(index.index!)
+        );
+
+        combinedDeprecations = combinedDeprecations.concat(withoutSystemIndices);
       } else {
         const deprecationsByType = deprecations[
           deprecationType as keyof MigrationDeprecationInfoResponse

x-pack/plugins/upgrade_assistant/server/lib/es_system_indices_upgrade.test.ts

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { convertFeaturesToIndicesArray } from './es_system_indices_upgrade';
+import { SystemIndicesUpgradeStatus } from '../../common/types';
+
+const esUpgradeSystemIndicesStatusMock: SystemIndicesUpgradeStatus = {
+  features: [
+    {
+      feature_name: 'machine_learning',
+      minimum_index_version: '7.1.1',
+      upgrade_status: 'UPGRADE_NEEDED',
+      indices: [
+        {
+          index: '.ml-config',
+          version: '7.1.1',
+        },
+        {
+          index: '.ml-notifications',
+          version: '7.1.1',
+        },
+      ],
+    },
+    {
+      feature_name: 'security',
+      minimum_index_version: '7.1.1',
+      upgrade_status: 'UPGRADE_NEEDED',
+      indices: [
+        {
+          index: '.ml-config',
+          version: '7.1.1',
+        },
+      ],
+    },
+  ],
+  upgrade_status: 'UPGRADE_NEEDED',
+};
+
+describe('convertFeaturesToIndicesArray', () => {
+  it('converts list with features to flat array of uniq indices', async () => {
+    const result = convertFeaturesToIndicesArray(esUpgradeSystemIndicesStatusMock.features);
+    expect(result).toEqual(['.ml-config', '.ml-notifications']);
+  });
+
+  it('returns empty array if no features are passed to it', async () => {
+    expect(convertFeaturesToIndicesArray([])).toEqual([]);
+  });
+});

x-pack/plugins/upgrade_assistant/server/lib/es_system_indices_upgrade.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { flow, flatMap, map, flatten, uniq } from 'lodash/fp';
+import { ElasticsearchClient } from 'src/core/server';
+import { SystemIndicesUpgradeStatus, SystemIndicesUpgradeFeature } from '../../common/types';
+
+export const convertFeaturesToIndicesArray = (
+  features: SystemIndicesUpgradeFeature[]
+): string[] => {
+  return flow(
+    // Map each feature into Indices[]
+    map('indices'),
+    // Flatten each into an string[] of indices
+    map(flatMap('index')),
+    // Flatten the array
+    flatten,
+    // And finally dedupe the indices
+    uniq
+  )(features);
+};
+
+export const getESSystemIndicesUpgradeStatus = async (
+  client: ElasticsearchClient
+): Promise<SystemIndicesUpgradeStatus> => {
+  const { body } = await client.transport.request({
+    method: 'GET',
+    path: '/_migration/system_features',
+  });
+
+  return body as SystemIndicesUpgradeStatus;
+};