From 3e74a2c80bae732ad11df711410f8156ec4e370a Mon Sep 17 00:00:00 2001
From: Tyler Smalley <tylersmalley@me.com>
Date: Wed, 7 Nov 2018 16:15:06 -0800
Subject: [PATCH] [migration] Retry attempts for ES service unavailable
 (#25255) (#25338)

Resolves a race condition where ES is technically up, but the Kibana
index is not.

Closes #25027

Signed-off-by: Tyler Smalley <tyler.smalley@elastic.co>
---
 .../migrations/core/elastic_index.ts          | 64 +++++++++++--------
 1 file changed, 38 insertions(+), 26 deletions(-)

diff --git a/src/server/saved_objects/migrations/core/elastic_index.ts b/src/server/saved_objects/migrations/core/elastic_index.ts
index a46836235709..2e28ca0ac5b0 100644
--- a/src/server/saved_objects/migrations/core/elastic_index.ts
+++ b/src/server/saved_objects/migrations/core/elastic_index.ts
@@ -150,39 +150,51 @@ export async function write(callCluster: CallCluster, index: string, docs: RawDo
 export async function migrationsUpToDate(
   callCluster: CallCluster,
   index: string,
-  migrationVersion: MigrationVersion
+  migrationVersion: MigrationVersion,
+  retryCount: number = 10
 ): Promise<boolean> {
-  const indexInfo = await fetchInfo(callCluster, index);
+  try {
+    const indexInfo = await fetchInfo(callCluster, index);
 
-  if (!_.get(indexInfo, 'mappings.doc.properties.migrationVersion')) {
-    return false;
-  }
+    if (!_.get(indexInfo, 'mappings.doc.properties.migrationVersion')) {
+      return false;
+    }
 
-  // If no migrations are actually defined, we're up to date!
-  if (Object.keys(migrationVersion).length <= 0) {
-    return true;
-  }
+    // If no migrations are actually defined, we're up to date!
+    if (Object.keys(migrationVersion).length <= 0) {
+      return true;
+    }
 
-  const { count } = await callCluster('count', {
-    body: {
-      query: {
-        bool: {
-          should: Object.entries(migrationVersion).map(([type, latestVersion]) => ({
-            bool: {
-              must: [
-                { exists: { field: type } },
-                { bool: { must_not: { term: { [`migrationVersion.${type}`]: latestVersion } } } },
-              ],
-            },
-          })),
+    const { count } = await callCluster('count', {
+      body: {
+        query: {
+          bool: {
+            should: Object.entries(migrationVersion).map(([type, latestVersion]) => ({
+              bool: {
+                must: [
+                  { exists: { field: type } },
+                  { bool: { must_not: { term: { [`migrationVersion.${type}`]: latestVersion } } } },
+                ],
+              },
+            })),
+          },
         },
       },
-    },
-    index,
-    type: ROOT_TYPE,
-  });
+      index,
+      type: ROOT_TYPE,
+    });
 
-  return count === 0;
+    return count === 0;
+  } catch (e) {
+    // retry for Service Unavailable
+    if (e.status !== 503 || retryCount === 0) {
+      throw e;
+    }
+
+    await new Promise(r => setTimeout(r, 1000));
+
+    return await migrationsUpToDate(callCluster, index, migrationVersion, retryCount - 1);
+  }
 }
 
 /**