[Monitoring] Metricbeat migration status logic (#34871)

* Initial logic and tests for this * Improve test * Some cleanup and adding api integration tests * Add comments * Account for no documents returned * Add detection logic and tests * Ensure these fields are optional * Update detection logic * Beats, logstash and apm can only possibly exist * Fix some issues with APM * Fix tests * Small updates based on needing to also include the filebeat index * Fix issue with the reduce * PR feedback * More PR feedback * Add additional tests * Add additional tests
2019-04-17 09:17:37 -04:00 · 2019-04-17 09:17:37 -04:00 · 33f1123dbe
parent 6fdad7f623
commit 33f1123dbe
50 changed files with 31934 additions and 51 deletions
--- a/x-pack/plugins/monitoring/common/constants.js
+++ b/x-pack/plugins/monitoring/common/constants.js
@ -161,3 +161,6 @@ export const INDEX_ALERTS = '.monitoring-alerts-6,.monitoring-alerts-7';
 export const INDEX_PATTERN_ELASTICSEARCH = '.monitoring-es-6-*,.monitoring-es-7-*';

 export const INDEX_PATTERN_FILEBEAT = 'filebeat-*';
+
+// This is the unique token that exists in monitoring indices collected by metricbeat
+export const METRICBEAT_INDEX_NAME_UNIQUE_TOKEN = '-mb-';
--- a/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.js
+++ b/x-pack/plugins/monitoring/server/lib/cluster/get_index_patterns.js
@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { prefixIndexPattern } from '../ccs_utils';
+import {
+  INDEX_PATTERN_ELASTICSEARCH,
+  INDEX_PATTERN_KIBANA,
+  INDEX_PATTERN_LOGSTASH,
+  INDEX_PATTERN_BEATS,
+  INDEX_ALERTS
+} from '../../../common/constants';
+
+export function getIndexPatterns(server, additionalPatterns = {}) {
+  // wildcard means to search _all_ clusters
+  const ccs = '*';
+  const config = server.config();
+  const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
+  const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs);
+  const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs);
+  const beatsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
+  const apmIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
+  const alertsIndex = prefixIndexPattern(config, INDEX_ALERTS, ccs);
+  const indexPatterns = {
+    esIndexPattern,
+    kbnIndexPattern,
+    lsIndexPattern,
+    beatsIndexPattern,
+    apmIndexPattern,
+    alertsIndex,
+    ...Object.keys(additionalPatterns).reduce((accum, varName) => {
+      return {
+        ...accum,
+        [varName]: prefixIndexPattern(config, additionalPatterns[varName], ccs),
+      };
+    }, {})
+  };
+
+  return indexPatterns;
+}
--- a/x-pack/plugins/monitoring/server/lib/setup/collection/test/get_collection_status.js
+++ b/x-pack/plugins/monitoring/server/lib/setup/collection/test/get_collection_status.js
@ -0,0 +1,202 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import sinon from 'sinon';
+import { getCollectionStatus } from '../';
+import { getIndexPatterns } from '../../../cluster/get_index_patterns';
+
+const mockReq = (searchResult = {}, msearchResult = { responses: [] }) => {
+  return {
+    server: {
+      config() {
+        return {
+          get: sinon.stub()
+            .withArgs('server.uuid').returns('kibana-1234')
+        };
+      },
+      plugins: {
+        elasticsearch: {
+          getCluster() {
+            return {
+              callWithRequest(_req, type) {
+                return Promise.resolve(type === 'search' ? searchResult : msearchResult);
+              }
+            };
+          }
+        }
+      },
+    },
+  };
+};
+
+describe('getCollectionStatus', () => {
+  it('should handle all stack products with internal monitoring', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [
+                { key: 'apm_1', beat_type: { buckets: [ { key: 'apm-server' }] } },
+                { key: 'beats_1' }
+              ] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(1);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.kibana.byUuid.kibana_1.isInternalCollector).to.be(true);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.apm.totalUniqueInstanceCount).to.be(1);
+    expect(result.apm.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.apm.byUuid.apm_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.elasticsearch.byUuid.es_1.isInternalCollector).to.be(true);
+  });
+
+  it('should handle some stack products as fully migrated', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-mb-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-mb-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [{ key: 'beats_1' }] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(1);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.kibana.byUuid.kibana_1.isFullyMigrated).to.be(true);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
+  });
+
+  it('should handle some stack products as partially migrated', async () => {
+    const req = mockReq({
+      aggregations: {
+        indices: {
+          buckets: [
+            {
+              key: '.monitoring-es-7-mb-2019',
+              es_uuids: { buckets: [{ key: 'es_1' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-mb-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1' }, { key: 'kibana_2' }] }
+            },
+            {
+              key: '.monitoring-kibana-7-2019',
+              kibana_uuids: { buckets: [{ key: 'kibana_1', by_timestamp: { value: 12 } }] }
+            },
+            {
+              key: '.monitoring-beats-7-2019',
+              beats_uuids: { buckets: [{ key: 'beats_1' }] }
+            },
+            {
+              key: '.monitoring-logstash-7-2019',
+              logstash_uuids: { buckets: [{ key: 'logstash_1' }] }
+            }
+          ]
+        }
+      }
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.totalUniqueInstanceCount).to.be(2);
+    expect(result.kibana.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.kibana.byUuid.kibana_1.isPartiallyMigrated).to.be(true);
+    expect(result.kibana.byUuid.kibana_1.lastInternallyCollectedTimestamp).to.be(12);
+
+    expect(result.beats.totalUniqueInstanceCount).to.be(1);
+    expect(result.beats.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.beats.byUuid.beats_1.isInternalCollector).to.be(true);
+
+    expect(result.logstash.totalUniqueInstanceCount).to.be(1);
+    expect(result.logstash.totalUniqueFullyMigratedCount).to.be(0);
+    expect(result.logstash.byUuid.logstash_1.isInternalCollector).to.be(true);
+
+    expect(result.elasticsearch.totalUniqueInstanceCount).to.be(1);
+    expect(result.elasticsearch.totalUniqueFullyMigratedCount).to.be(1);
+    expect(result.elasticsearch.byUuid.es_1.isFullyMigrated).to.be(true);
+  });
+
+  it('should detect products based on other indices', async () => {
+    const req = mockReq({}, {
+      responses: [
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } },
+        { hits: { total: { value: 1 } } }
+      ]
+    });
+
+    const result = await getCollectionStatus(req, getIndexPatterns(req.server));
+
+    expect(result.kibana.detected.doesExist).to.be(true);
+    expect(result.elasticsearch.detected.doesExist).to.be(true);
+    expect(result.beats.detected.mightExist).to.be(true);
+    expect(result.logstash.detected.mightExist).to.be(true);
+  });
+});
--- a/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.js
+++ b/x-pack/plugins/monitoring/server/lib/setup/collection/get_collection_status.js
@ -0,0 +1,382 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { get, uniq } from 'lodash';
+import { METRICBEAT_INDEX_NAME_UNIQUE_TOKEN } from '../../../../common/constants';
+import { KIBANA_SYSTEM_ID, BEATS_SYSTEM_ID, LOGSTASH_SYSTEM_ID } from '../../../../../xpack_main/common/constants';
+
+const APM_CUSTOM_ID = 'apm';
+const ELASTICSEARCH_CUSTOM_ID = 'elasticsearch';
+
+const getRecentMonitoringDocuments = async (req, indexPatterns, clusterUuid) => {
+  const start = get(req.payload, 'timeRange.min', 'now-30s');
+  const end = get(req.payload, 'timeRange.max', 'now');
+
+  const filters = [
+    {
+      range: {
+        'timestamp': {
+          gte: start,
+          lte: end
+        }
+      }
+    }
+  ];
+
+  if (clusterUuid) {
+    filters.push({ term: { 'cluster_uuid': clusterUuid } });
+  }
+
+  const params = {
+    index: Object.values(indexPatterns),
+    size: 0,
+    ignoreUnavailable: true,
+    filterPath: [
+      'aggregations.indices.buckets'
+    ],
+    body: {
+      query: {
+        bool: {
+          filter: filters,
+        }
+      },
+      aggs: {
+        indices: {
+          terms: {
+            field: '_index',
+            size: 50,
+          },
+          aggs: {
+            es_uuids: {
+              terms: {
+                field: 'node_stats.node_id'
+              },
+              aggs: {
+                by_timestamp: {
+                  max: {
+                    field: 'timestamp'
+                  }
+                }
+              }
+            },
+            kibana_uuids: {
+              terms: {
+                field: 'kibana_stats.kibana.uuid'
+              },
+              aggs: {
+                by_timestamp: {
+                  max: {
+                    field: 'timestamp'
+                  }
+                }
+              }
+            },
+            beats_uuids: {
+              terms: {
+                field: 'beats_stats.beat.uuid'
+              },
+              aggs: {
+                by_timestamp: {
+                  max: {
+                    field: 'timestamp'
+                  }
+                },
+                beat_type: {
+                  terms: {
+                    field: 'beats_stats.beat.type'
+                  }
+                }
+              }
+            },
+            logstash_uuids: {
+              terms: {
+                field: 'logstash_stats.logstash.uuid'
+              },
+              aggs: {
+                by_timestamp: {
+                  max: {
+                    field: 'timestamp'
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  };
+
+  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
+  return await callWithRequest(req, 'search', params);
+};
+
+async function detectProducts(req) {
+  const result = {
+    [KIBANA_SYSTEM_ID]: {
+      doesExist: true,
+    },
+    [ELASTICSEARCH_CUSTOM_ID]: {
+      doesExist: true,
+    },
+    [BEATS_SYSTEM_ID]: {
+      mightExist: false,
+    },
+    [APM_CUSTOM_ID]: {
+      mightExist: false,
+    },
+    [LOGSTASH_SYSTEM_ID]: {
+      mightExist: false,
+    }
+  };
+
+  const msearch = [
+    { index: '*beat-*' },
+    { size: 0, terminate_after: 0 },
+
+    { index: '.management-beats*' },
+    { size: 0, terminate_after: 0 },
+
+    { index: 'logstash-*' },
+    { size: 0, terminate_after: 0 },
+
+    { index: '.logstash*' },
+    { size: 0, terminate_after: 0 },
+
+    { index: 'apm*' },
+    { size: 0, terminate_after: 0 },
+  ];
+
+  const { callWithRequest } = req.server.plugins.elasticsearch.getCluster('monitoring');
+  const {
+    responses: [
+      beatsDataDetectionResponse,
+      beatsManagementDetectionResponse,
+      logstashDataDetectionResponse,
+      logstashManagementDetectionResponse,
+      apmDetectionResponse
+    ]
+  } = await callWithRequest(req, 'msearch', { body: msearch });
+
+  if (get(beatsDataDetectionResponse, 'hits.total.value', 0) > 0
+    || get(beatsManagementDetectionResponse, 'hits.total.value', 0) > 0) {
+    result[BEATS_SYSTEM_ID].mightExist = true;
+  }
+
+  if (get(logstashDataDetectionResponse, 'hits.total.value', 0) > 0
+    || get(logstashManagementDetectionResponse, 'hits.total.value', 0) > 0) {
+    result[LOGSTASH_SYSTEM_ID].mightExist = true;
+  }
+
+  if (get(apmDetectionResponse, 'hits.total.value', 0) > 0) {
+    result[APM_CUSTOM_ID].mightExist = true;
+  }
+
+  return result;
+}
+
+function getUuidBucketName(productName) {
+  switch (productName) {
+    case ELASTICSEARCH_CUSTOM_ID:
+      return 'es_uuids';
+    case KIBANA_SYSTEM_ID:
+      return 'kibana_uuids';
+    case BEATS_SYSTEM_ID:
+    case APM_CUSTOM_ID:
+      return 'beats_uuids';
+    case LOGSTASH_SYSTEM_ID:
+      return 'logstash_uuids';
+  }
+}
+
+function isBeatFromAPM(bucket) {
+  const beatType = get(bucket, 'beat_type');
+  if (!beatType) {
+    return false;
+  }
+
+  return get(beatType, 'buckets[0].key') === 'apm-server';
+}
+
+/**
+ * Determines if we should ignore this bucket from this product.
+ *
+ * We need this logic because APM and Beats are separate products, but their
+ * monitoring data appears in the same index (.monitoring-beats-*) and the single
+ * way to determine the difference between two documents in that index
+ * is `beats_stats.beat.type` which we are performing a terms agg in the above query.
+ * If that value is `apm-server` and we're attempting to calculating status for beats
+ * we need to ignore that data from that particular  bucket.
+ *
+ * @param {*} product The product object, which are stored in PRODUCTS
+ * @param {*} bucket The agg bucket in the response
+ */
+function shouldSkipBucket(product, bucket) {
+  if (product.name === BEATS_SYSTEM_ID && isBeatFromAPM(bucket)) {
+    return true;
+  }
+  if (product.name === APM_CUSTOM_ID && !isBeatFromAPM(bucket)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * This function will scan all monitoring documents within the past 30s (or a custom time range is supported too)
+ * and determine which products fall into one of three states:
+ * - isPartiallyMigrated: This means we are seeing some monitoring documents from MB and some from internal collection
+ * - isFullyMigrated: This means we are only seeing monitoring documents from MB
+ * - isInternalCollector: This means we are only seeing monitoring documents from internal collection
+ *
+ * If a product is partially migrated, this function will also return the timestamp of the last seen monitoring
+ * document from internal collection. This will help the user understand if they successfully disabled internal
+ * collection and just need to wait for the time window of the query to exclude the older, internally collected documents
+ *
+ * If a product is not detected at all (no monitoring documents), we will attempt to do some self discovery
+ * based on assumptions around indices that might exist with various products. We will return something
+ * like this in that case:
+ * detected: {
+ *   doesExist: boolean, // This product definitely exists but does not have any monitoring documents (kibana and ES)
+ *   mightExist: boolean, // This product might exist based on the presence of some other indices
+ * }
+
+ * @param {*} req Standard request object. Can contain a timeRange to use for the query
+ * @param {*} indexPatterns Map of index patterns to search against (will be all .monitoring-* indices)
+ * @param {*} clusterUuid Optional and will be used to filter down the query if used
+ */
+export const getCollectionStatus = async (req, indexPatterns, clusterUuid) => {
+  const PRODUCTS = [
+    { name: KIBANA_SYSTEM_ID },
+    { name: BEATS_SYSTEM_ID },
+    { name: LOGSTASH_SYSTEM_ID },
+    { name: APM_CUSTOM_ID, token: '-beats-' },
+    { name: ELASTICSEARCH_CUSTOM_ID, token: '-es-' },
+  ];
+
+  const [
+    recentDocuments,
+    detectedProducts
+  ] = await Promise.all([
+    await getRecentMonitoringDocuments(req, indexPatterns, clusterUuid),
+    await detectProducts(req)
+  ]);
+
+  const indicesBuckets = get(recentDocuments, 'aggregations.indices.buckets', []);
+
+  const status = PRODUCTS.reduce((products, product) => {
+    const token = product.token || product.name;
+    const indexBuckets = indicesBuckets.filter(bucket => bucket.key.includes(token));
+    const uuidBucketName = getUuidBucketName(product.name);
+
+    const productStatus = {
+      totalUniqueInstanceCount: 0,
+      totalUniqueFullyMigratedCount: 0,
+      detected: null,
+      byUuid: null,
+    };
+
+    const fullyMigratedUuidsMap = {};
+    const internalCollectorsUuidsMap = {};
+    const partiallyMigratedUuidsMap = {};
+
+    // If there is no data, then they are a net new user
+    if (!indexBuckets || indexBuckets.length === 0) {
+      productStatus.totalUniqueInstanceCount = 0;
+      productStatus.detected = detectedProducts[product.name];
+    }
+    // If there is a single bucket, then they are fully migrated or fully on the internal collector
+    else if (indexBuckets.length === 1) {
+      const singleIndexBucket = indexBuckets[0];
+      const isFullyMigrated = singleIndexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN);
+
+
+      const map = isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap;
+      const uuidBuckets = get(singleIndexBucket, `${uuidBucketName}.buckets`, []);
+      for (const bucket of uuidBuckets) {
+        if (shouldSkipBucket(product, bucket)) {
+          continue;
+        }
+        const { key, by_timestamp: byTimestamp } = bucket;
+        if (!map[key]) {
+          map[key] = { lastTimestamp: get(byTimestamp, 'value') };
+        }
+      }
+      productStatus.totalUniqueInstanceCount = Object.keys(map).length;
+      productStatus.totalUniqueFullyMigratedCount = Object.keys(fullyMigratedUuidsMap).length;
+      productStatus.byUuid = {
+        ...Object.keys(internalCollectorsUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isInternalCollector: true, ...internalCollectorsUuidsMap[uuid] }
+        }), {}),
+        ...Object.keys(partiallyMigratedUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isPartiallyMigrated: true, ...partiallyMigratedUuidsMap[uuid] }
+        }), {}),
+        ...Object.keys(fullyMigratedUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isFullyMigrated: true, ...fullyMigratedUuidsMap[uuid] }
+        }), {}),
+      };
+    }
+    // If there are multiple buckets, they are partially upgraded assuming a single mb index exists
+    else {
+      const internalTimestamps = [];
+      for (const indexBucket of indexBuckets) {
+        const isFullyMigrated = indexBucket.key.includes(METRICBEAT_INDEX_NAME_UNIQUE_TOKEN);
+        const map = isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap;
+        const otherMap = !isFullyMigrated ? fullyMigratedUuidsMap : internalCollectorsUuidsMap;
+
+        const uuidBuckets = get(indexBucket, `${uuidBucketName}.buckets`, []);
+        for (const bucket of uuidBuckets) {
+          if (shouldSkipBucket(product, bucket)) {
+            continue;
+          }
+
+          const { key, by_timestamp: byTimestamp } = bucket;
+          if (!map[key]) {
+            if (otherMap[key]) {
+              partiallyMigratedUuidsMap[key] = true;
+              delete otherMap[key];
+            }
+            else {
+              map[key] = true;
+            }
+          }
+          if (!isFullyMigrated) {
+            internalTimestamps.push(byTimestamp.value);
+          }
+        }
+      }
+
+      productStatus.totalUniqueInstanceCount = uniq([
+        ...Object.keys(internalCollectorsUuidsMap),
+        ...Object.keys(fullyMigratedUuidsMap),
+        ...Object.keys(partiallyMigratedUuidsMap)
+      ]).length;
+      productStatus.totalUniqueFullyMigratedCount = Object.keys(fullyMigratedUuidsMap).length;
+      productStatus.byUuid = {
+        ...Object.keys(internalCollectorsUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isInternalCollector: true }
+        }), {}),
+        ...Object.keys(partiallyMigratedUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isPartiallyMigrated: true, lastInternallyCollectedTimestamp: internalTimestamps[0] }
+        }), {}),
+        ...Object.keys(fullyMigratedUuidsMap).reduce((accum, uuid) => ({
+          ...accum,
+          [uuid]: { isFullyMigrated: true }
+        }), {}),
+      };
+    }
+
+    return {
+      ...products,
+      [product.name]: productStatus,
+    };
+  }, {});
+
+  return status;
+};
--- a/x-pack/plugins/monitoring/server/lib/setup/collection/index.js
+++ b/x-pack/plugins/monitoring/server/lib/setup/collection/index.js
@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { getCollectionStatus } from './get_collection_status';
--- a/x-pack/plugins/monitoring/server/routes/api/v1/cluster/cluster.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/cluster/cluster.js
@ -7,13 +7,8 @@
 import Joi from 'joi';
 import { getClustersFromRequest } from '../../../../lib/cluster/get_clusters_from_request';
 import { handleError } from '../../../../lib/errors';
-import { prefixIndexPattern } from '../../../../lib/ccs_utils';
+import { getIndexPatterns } from '../../../../lib/cluster/get_index_patterns';
 import {
-  INDEX_PATTERN_KIBANA,
-  INDEX_PATTERN_ELASTICSEARCH,
-  INDEX_PATTERN_LOGSTASH,
-  INDEX_PATTERN_BEATS,
-  INDEX_ALERTS,
  INDEX_PATTERN_FILEBEAT
 } from '../../../../../common/constants';

@ -39,24 +34,7 @@ export function clusterRoute(server) {
      }
    },
    handler: (req) => {
-      const config = server.config();
-      const ccs = req.payload.ccs;
-      const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
-      const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs);
-      const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs);
-      const beatsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
-      const apmIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
-      const alertsIndex = prefixIndexPattern(config, INDEX_ALERTS, ccs);
-      const filebeatIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_FILEBEAT, '*');
-      const indexPatterns = {
-        esIndexPattern,
-        kbnIndexPattern,
-        lsIndexPattern,
-        beatsIndexPattern,
-        apmIndexPattern,
-        alertsIndex,
-        filebeatIndexPattern
-      };
+      const indexPatterns = getIndexPatterns(server, { filebeatIndexPattern: INDEX_PATTERN_FILEBEAT });
      const options = {
        clusterUuid: req.params.clusterUuid,
        start: req.payload.timeRange.min,
--- a/x-pack/plugins/monitoring/server/routes/api/v1/cluster/clusters.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/cluster/clusters.js
@ -8,15 +8,10 @@ import Joi from 'joi';
 import { getClustersFromRequest } from '../../../../lib/cluster/get_clusters_from_request';
 import { verifyMonitoringAuth } from '../../../../lib/elasticsearch/verify_monitoring_auth';
 import { handleError } from '../../../../lib/errors';
-import { prefixIndexPattern } from '../../../../lib/ccs_utils';
 import {
-  INDEX_PATTERN_ELASTICSEARCH,
-  INDEX_PATTERN_KIBANA,
-  INDEX_PATTERN_LOGSTASH,
-  INDEX_PATTERN_BEATS,
-  INDEX_ALERTS,
  INDEX_PATTERN_FILEBEAT
 } from '../../../../../common/constants';
+import { getIndexPatterns } from '../../../../lib/cluster/get_index_patterns';

 export function clustersRoute(server) {
  /*
@ -44,27 +39,7 @@ export function clustersRoute(server) {
      // the monitoring data. `try/catch` makes it a little more explicit.
      try {
        await verifyMonitoringAuth(req);
-
-        // wildcard means to search _all_ clusters
-        const ccs = '*';
-        const config = server.config();
-        const esIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_ELASTICSEARCH, ccs);
-        const kbnIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_KIBANA, ccs);
-        const lsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_LOGSTASH, ccs);
-        const beatsIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
-        const apmIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_BEATS, ccs);
-        const alertsIndex = prefixIndexPattern(config, INDEX_ALERTS, ccs);
-        const filebeatIndexPattern = prefixIndexPattern(config, INDEX_PATTERN_FILEBEAT, ccs);
-        const indexPatterns = {
-          esIndexPattern,
-          kbnIndexPattern,
-          lsIndexPattern,
-          beatsIndexPattern,
-          apmIndexPattern,
-          alertsIndex,
-          filebeatIndexPattern
-        };
-
+        const indexPatterns = getIndexPatterns(server, { filebeatIndexPattern: INDEX_PATTERN_FILEBEAT });
        clusters = await getClustersFromRequest(req, indexPatterns);
      } catch (err) {
        throw handleError(err, req);
--- a/x-pack/plugins/monitoring/server/routes/api/v1/setup/cluster_setup_status.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/setup/cluster_setup_status.js
@ -0,0 +1,50 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import Joi from 'joi';
+import { verifyMonitoringAuth } from '../../../../lib/elasticsearch/verify_monitoring_auth';
+import { handleError } from '../../../../lib/errors';
+import { getCollectionStatus } from '../../../../lib/setup/collection';
+import { getIndexPatterns } from '../../../../lib/cluster/get_index_patterns';
+
+export function clusterSetupStatusRoute(server) {
+  /*
+   * Monitoring Home
+   * Route Init (for checking license and compatibility for multi-cluster monitoring
+   */
+  server.route({
+    method: 'POST',
+    path: '/api/monitoring/v1/setup/collection/{clusterUuid}',
+    config: {
+      validate: {
+        params: Joi.object({
+          clusterUuid: Joi.string().required()
+        }),
+        payload: Joi.object({
+          timeRange: Joi.object({
+            min: Joi.date().required(),
+            max: Joi.date().required()
+          }).optional()
+        }).allow(null)
+      }
+    },
+    handler: async (req) => {
+      let status = null;
+
+      // NOTE using try/catch because checkMonitoringAuth is expected to throw
+      // an error when current logged-in user doesn't have permission to read
+      // the monitoring data. `try/catch` makes it a little more explicit.
+      try {
+        await verifyMonitoringAuth(req);
+        const indexPatterns = getIndexPatterns(server);
+        status = await getCollectionStatus(req, indexPatterns, req.params.clusterUuid);
+      } catch (err) {
+        throw handleError(err, req);
+      }
+
+      return status;
+    }
+  });
+}
--- a/x-pack/plugins/monitoring/server/routes/api/v1/setup/clusters_setup_status.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/setup/clusters_setup_status.js
@ -0,0 +1,47 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import Joi from 'joi';
+import { verifyMonitoringAuth } from '../../../../lib/elasticsearch/verify_monitoring_auth';
+import { handleError } from '../../../../lib/errors';
+import { getCollectionStatus } from '../../../../lib/setup/collection';
+import { getIndexPatterns } from '../../../../lib/cluster/get_index_patterns';
+
+export function clustersSetupStatusRoute(server) {
+  /*
+   * Monitoring Home
+   * Route Init (for checking license and compatibility for multi-cluster monitoring
+   */
+  server.route({
+    method: 'POST',
+    path: '/api/monitoring/v1/setup/collection',
+    config: {
+      validate: {
+        payload: Joi.object({
+          timeRange: Joi.object({
+            min: Joi.date().required(),
+            max: Joi.date().required()
+          }).optional()
+        }).allow(null)
+      }
+    },
+    handler: async (req) => {
+      let status = null;
+
+      // NOTE using try/catch because checkMonitoringAuth is expected to throw
+      // an error when current logged-in user doesn't have permission to read
+      // the monitoring data. `try/catch` makes it a little more explicit.
+      try {
+        await verifyMonitoringAuth(req);
+        const indexPatterns = getIndexPatterns(server);
+        status = await getCollectionStatus(req, indexPatterns);
+      } catch (err) {
+        throw handleError(err, req);
+      }
+
+      return status;
+    }
+  });
+}
--- a/x-pack/plugins/monitoring/server/routes/api/v1/setup/index.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/setup/index.js
@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { clustersSetupStatusRoute } from './clusters_setup_status';
+export { clusterSetupStatusRoute } from './cluster_setup_status';
--- a/x-pack/plugins/monitoring/server/routes/api/v1/ui.js
+++ b/x-pack/plugins/monitoring/server/routes/api/v1/ui.js
@ -54,3 +54,4 @@ export {
  logstashOverviewRoute,
  logstashPipelineRoute
 } from './logstash';
+export * from './setup';
--- a/x-pack/test/api_integration/apis/monitoring/index.js
+++ b/x-pack/test/api_integration/apis/monitoring/index.js
@ -16,5 +16,6 @@ export default function ({ loadTestFile }) {
    loadTestFile(require.resolve('./common'));
    loadTestFile(require.resolve('./standalone_cluster'));
    loadTestFile(require.resolve('./logs'));
+    loadTestFile(require.resolve('./setup'));
  });
 }
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_apm.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_apm.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/detect_apm';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('detect_apm', () => {
+    const archive = 'monitoring/setup/collection/detect_apm';
+    const timeRange = {
+      min: '2019-04-16T00:00:00.741Z',
+      max: '2019-04-16T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_beats.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_beats.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/detect_beats';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('detect_beats', () => {
+    const archive = 'monitoring/setup/collection/detect_beats';
+    const timeRange = {
+      min: '2019-04-09T00:00:00.741Z',
+      max: '2019-04-09T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_beats_management.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_beats_management.js
@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/detect_beats_management';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('detect_beats_management', () => {
+    const archive = 'monitoring/setup/collection/detect_beats_management';
+    const timeRange = {
+      min: '2019-04-16T00:00:00.741Z',
+      max: '2019-04-16T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_logstash.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_logstash.js
@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/detect_logstash';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('detect_logstash', () => {
+    const archive = 'monitoring/setup/collection/detect_logstash';
+    const timeRange = {
+      min: '2019-04-16T00:00:00.741Z',
+      max: '2019-04-16T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_logstash_management.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/detect_logstash_management.js
@ -0,0 +1,38 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/detect_logstash_management';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('detect_logstash_management', () => {
+    const archive = 'monitoring/setup/collection/detect_logstash_management';
+    const timeRange = {
+      min: '2019-04-16T00:00:00.741Z',
+      max: '2019-04-16T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/es_and_kibana_exclusive_mb.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/es_and_kibana_exclusive_mb.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/es_and_kibana_exclusive_mb';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('es_and_kibana_exclusive_mb', () => {
+    const archive = 'monitoring/setup/collection/es_and_kibana_exclusive_mb';
+    const timeRange = {
+      min: '2019-04-09T00:00:00.741Z',
+      max: '2019-04-09T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/es_and_kibana_mb.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/es_and_kibana_mb.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/es_and_kibana_mb';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('es_and_kibana_mb', () => {
+    const archive = 'monitoring/setup/collection/es_and_kibana_mb';
+    const timeRange = {
+      min: '2019-04-09T00:00:00.741Z',
+      max: '2019-04-09T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_apm.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_apm.json
@ -0,0 +1,42 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": true
+    },
+    "byUuid": null
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_beats.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_beats.json
@ -0,0 +1,48 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 1,
+    "detected": null,
+    "byUuid": {
+      "5b2de169-2785-441b-ae8c-186a1936b17d": {
+        "isFullyMigrated": true,
+        "lastTimestamp": 1554841069921
+      }
+    }
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": true
+    },
+    "byUuid": null
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "agI8JhXhShasvuDgq0VxRg": {
+        "isPartiallyMigrated": true,
+        "lastInternallyCollectedTimestamp": 1554841077638
+      }
+    }
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_beats_management.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_beats_management.json
@ -0,0 +1,42 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": true
+    },
+    "byUuid": null
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_logstash.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_logstash.json
@ -0,0 +1,42 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": true
+    },
+    "byUuid": null
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_logstash_management.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/detect_logstash_management.json
@ -0,0 +1,42 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": true
+    },
+    "byUuid": null
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "mightExist": false
+    },
+    "byUuid": null
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": {
+      "doesExist": true
+    },
+    "byUuid": null
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/es_and_kibana_exclusive_mb.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/es_and_kibana_exclusive_mb.json
@ -0,0 +1,52 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 1,
+    "detected": null,
+    "byUuid": {
+      "5b2de169-2785-441b-ae8c-186a1936b17d": {
+        "isFullyMigrated": true,
+        "lastTimestamp": 1554821587077
+      }
+    }
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "8eba4902-df80-43b0-b6c2-ed8ca290984e": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821586714
+      }
+    }
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {}
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "4134a00e-89e4-4896-a3d4-c3a9aa03a594": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821579833
+      }
+    }
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 1,
+    "detected": null,
+    "byUuid": {
+      "agI8JhXhShasvuDgq0VxRg": {
+        "isFullyMigrated": true,
+        "lastTimestamp": 1554821579822
+      }
+    }
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/es_and_kibana_mb.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/es_and_kibana_mb.json
@ -0,0 +1,52 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "5b2de169-2785-441b-ae8c-186a1936b17d": {
+        "isPartiallyMigrated": true,
+        "lastInternallyCollectedTimestamp": 1554821412725
+      }
+    }
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "8eba4902-df80-43b0-b6c2-ed8ca290984e": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821406717
+      }
+    }
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {}
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "4134a00e-89e4-4896-a3d4-c3a9aa03a594": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821409656
+      }
+    }
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "agI8JhXhShasvuDgq0VxRg": {
+        "isPartiallyMigrated": true,
+        "lastInternallyCollectedTimestamp": 1554821411520
+      }
+    }
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/kibana_exclusive_mb.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/kibana_exclusive_mb.json
@ -0,0 +1,52 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 1,
+    "detected": null,
+    "byUuid": {
+      "5b2de169-2785-441b-ae8c-186a1936b17d": {
+        "isFullyMigrated": true,
+        "lastTimestamp": 1554821537079
+      }
+    }
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "8eba4902-df80-43b0-b6c2-ed8ca290984e": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821536716
+      }
+    }
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {}
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "4134a00e-89e4-4896-a3d4-c3a9aa03a594": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821539784
+      }
+    }
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "agI8JhXhShasvuDgq0VxRg": {
+        "isPartiallyMigrated": true,
+        "lastInternallyCollectedTimestamp": 1554821531545
+      }
+    }
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/kibana_mb.json
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/fixtures/kibana_mb.json
@ -0,0 +1,52 @@
+{
+  "kibana": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "5b2de169-2785-441b-ae8c-186a1936b17d": {
+        "isPartiallyMigrated": true,
+        "lastInternallyCollectedTimestamp": 1554821352739
+      }
+    }
+  },
+  "beats": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "8eba4902-df80-43b0-b6c2-ed8ca290984e": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821354041
+      }
+    }
+  },
+  "apm": {
+    "totalUniqueInstanceCount": 0,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {}
+  },
+  "logstash": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "4134a00e-89e4-4896-a3d4-c3a9aa03a594": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821359616
+      }
+    }
+  },
+  "elasticsearch": {
+    "totalUniqueInstanceCount": 1,
+    "totalUniqueFullyMigratedCount": 0,
+    "detected": null,
+    "byUuid": {
+      "agI8JhXhShasvuDgq0VxRg": {
+        "isInternalCollector": true,
+        "lastTimestamp": 1554821351499
+      }
+    }
+  }
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/index.js
@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('Collection', () => {
+    loadTestFile(require.resolve('./kibana_mb'));
+    loadTestFile(require.resolve('./kibana_exclusive_mb'));
+    loadTestFile(require.resolve('./es_and_kibana_mb'));
+    loadTestFile(require.resolve('./es_and_kibana_exclusive_mb'));
+    loadTestFile(require.resolve('./detect_beats'));
+    loadTestFile(require.resolve('./detect_beats_management'));
+    loadTestFile(require.resolve('./detect_logstash'));
+    loadTestFile(require.resolve('./detect_logstash_management'));
+    loadTestFile(require.resolve('./detect_apm'));
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/kibana_exclusive_mb.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/kibana_exclusive_mb.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/kibana_exclusive_mb';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('kibana_exclusive_mb', () => {
+    const archive = 'monitoring/setup/collection/kibana_exclusive_mb';
+    const timeRange = {
+      min: '2019-04-09T00:00:00.741Z',
+      max: '2019-04-09T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/collection/kibana_mb.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/collection/kibana_mb.js
@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import expect from '@kbn/expect';
+import fixture from './fixtures/kibana_mb';
+
+export default function ({ getService }) {
+  const supertest = getService('supertest');
+  const esArchiver = getService('esArchiver');
+
+  describe('kibana_mb', () => {
+    const archive = 'monitoring/setup/collection/kibana_mb';
+    const timeRange = {
+      min: '2019-04-09T00:00:00.741Z',
+      max: '2019-04-09T23:59:59.741Z'
+    };
+
+    before('load archive', () => {
+      return esArchiver.load(archive);
+    });
+
+    after('unload archive', () => {
+      return esArchiver.unload(archive);
+    });
+
+    it('should get collection status', async () => {
+      const { body } = await supertest
+        .post('/api/monitoring/v1/setup/collection')
+        .set('kbn-xsrf', 'xxx')
+        .send({ timeRange })
+        .expect(200);
+
+      expect(body).to.eql(fixture);
+    });
+  });
+}
--- a/x-pack/test/api_integration/apis/monitoring/setup/index.js
+++ b/x-pack/test/api_integration/apis/monitoring/setup/index.js
@ -0,0 +1,11 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default function ({ loadTestFile }) {
+  describe('Setup', () => {
+    loadTestFile(require.resolve('./collection'));
+  });
+}
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_apm/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_apm/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_apm/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_apm/mappings.json
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats/mappings.json
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats_management/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats_management/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats_management/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_beats_management/mappings.json
@ -0,0 +1,142 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": ".management-beats",
+    "mappings": {
+      "dynamic": "strict",
+      "properties": {
+        "beat": {
+          "properties": {
+            "access_token": {
+              "type": "keyword"
+            },
+            "active": {
+              "type": "boolean"
+            },
+            "enrollment_token": {
+              "type": "keyword"
+            },
+            "ephemeral_id": {
+              "type": "keyword"
+            },
+            "host_ip": {
+              "type": "ip"
+            },
+            "host_name": {
+              "type": "keyword"
+            },
+            "id": {
+              "type": "keyword"
+            },
+            "last_checkin": {
+              "type": "date"
+            },
+            "metadata": {
+              "dynamic": "true",
+              "type": "object"
+            },
+            "name": {
+              "type": "keyword"
+            },
+            "status": {
+              "properties": {
+                "event": {
+                  "properties": {
+                    "message": {
+                      "type": "text"
+                    },
+                    "type": {
+                      "type": "keyword"
+                    },
+                    "uuid": {
+                      "type": "keyword"
+                    }
+                  }
+                },
+                "timestamp": {
+                  "type": "date"
+                },
+                "type": {
+                  "type": "keyword"
+                }
+              }
+            },
+            "tags": {
+              "type": "keyword"
+            },
+            "type": {
+              "type": "keyword"
+            },
+            "verified_on": {
+              "type": "date"
+            },
+            "version": {
+              "type": "keyword"
+            }
+          }
+        },
+        "configuration_block": {
+          "properties": {
+            "config": {
+              "type": "keyword"
+            },
+            "description": {
+              "type": "text"
+            },
+            "id": {
+              "type": "keyword"
+            },
+            "last_updated": {
+              "type": "date"
+            },
+            "tag": {
+              "type": "keyword"
+            },
+            "type": {
+              "type": "keyword"
+            }
+          }
+        },
+        "enrollment_token": {
+          "properties": {
+            "expires_on": {
+              "type": "date"
+            },
+            "token": {
+              "type": "keyword"
+            }
+          }
+        },
+        "tag": {
+          "properties": {
+            "color": {
+              "type": "keyword"
+            },
+            "hasConfigurationBlocksTypes": {
+              "type": "keyword"
+            },
+            "id": {
+              "type": "keyword"
+            },
+            "name": {
+              "type": "keyword"
+            }
+          }
+        },
+        "type": {
+          "type": "keyword"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "auto_expand_replicas": "0-1",
+        "codec": "best_compression",
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash/mappings.json
@ -0,0 +1,54 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+      "logstash": {
+        "is_write_index": true
+      }
+    },
+    "index": "logstash-2019.04.16-000001",
+    "mappings": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "type": "text"
+        },
+        "host": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "type": "text"
+        },
+        "message": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "type": "text"
+        },
+        "sequence": {
+          "type": "long"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "number_of_replicas": "1",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash_management/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash_management/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash_management/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/detect_logstash_management/mappings.json
@ -0,0 +1,54 @@
+{
+  "type": "index",
+  "value": {
+    "aliases": {
+    },
+    "index": ".logstash",
+    "mappings": {
+      "_meta": {
+        "logstash-version": "8.0.0"
+      },
+      "dynamic": "strict",
+      "properties": {
+        "description": {
+          "type": "text"
+        },
+        "last_modified": {
+          "type": "date"
+        },
+        "metadata": {
+          "dynamic": "false",
+          "type": "object"
+        },
+        "pipeline": {
+          "type": "text"
+        },
+        "pipeline_metadata": {
+          "properties": {
+            "type": {
+              "type": "keyword"
+            },
+            "version": {
+              "type": "short"
+            }
+          }
+        },
+        "pipeline_settings": {
+          "dynamic": "false",
+          "type": "object"
+        },
+        "username": {
+          "type": "keyword"
+        }
+      }
+    },
+    "settings": {
+      "index": {
+        "auto_expand_replicas": "0-1",
+        "codec": "best_compression",
+        "number_of_replicas": "0",
+        "number_of_shards": "1"
+      }
+    }
+  }
+}
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_exclusive_mb/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_exclusive_mb/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_exclusive_mb/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_exclusive_mb/mappings.json
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_mb/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_mb/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_mb/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/es_and_kibana_mb/mappings.json
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_exclusive_mb/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_exclusive_mb/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_exclusive_mb/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_exclusive_mb/mappings.json
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_mb/data.json.gz
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_mb/data.json.gz
--- a/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_mb/mappings.json
+++ b/x-pack/test/functional/es_archives/monitoring/setup/collection/kibana_mb/mappings.json