[retry] implement waitFor method (#21747)

We currently use the `retry` service to call a function over and over in a loop, waiting for it to run without throwing an error, and ultimately failing if it does not succeed before a timeout is exceeded. This is the easiest way to get certain interactions to work, either because we don't know when we should be able to execute the interaction successfully, or because the timing is just too tricky to plan out correctly. Another place where we are using the `retry` service, which I don't think is appropriate, is when we need to wait for a certain condition to be met. This is where `retry.waitFor()` comes in:

```js
await retry.waitFor('dashboard search to be enabled', async () => {
  const searchInput = await testSubjects.find('savedObjectFinderSearchInput');
  return await searchInput.isEnabled();
})
```

The `retry.waitFor()` method behaves much like the `retry.try()` method behind the scenes, calling a function over and over, but instead of waiting for it to run without throwing an error it waits for it to return a "truthy" value. It also requires a description string that is used to make rather nice log output and a more descriptive error message than something like https://github.com/elastic/kibana/blob/master/test/functional/apps/dashboard/_data_shared_attributes.js#L61-L67

src/functional_test_runner/lib/config/schema.js

@@ -73,6 +73,7 @@ export const schema = Joi.object().keys({
   timeouts: Joi.object().keys({
     find: Joi.number().default(10000),
     try: Joi.number().default(40000),
+    waitFor: Joi.number().default(20000),
     esRequestTimeout: Joi.number().default(30000),
     kibanaStabilize: Joi.number().default(15000),
     navigateStatusPageCheck: Joi.number().default(250),

test/common/services/retry.js

@@ -1,68 +0,0 @@
-/*
- * Licensed to Elasticsearch B.V. under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch B.V. licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import bluebird from 'bluebird';
-
-export function RetryProvider({ getService }) {
-  const config = getService('config');
-  const log = getService('log');
-
-  class Retry {
-    tryForTime(timeout, block) {
-      const start = Date.now();
-      const retryDelay = 502;
-      let lastTry = 0;
-      let finalMessage;
-      let prevMessage;
-
-      function attempt() {
-        lastTry = Date.now();
-
-        if (lastTry - start > timeout) {
-          throw new Error('tryForTime timeout: ' + finalMessage);
-        }
-
-        return bluebird
-          .try(block)
-          .catch(function tryForTimeCatch(err) {
-            if (err.message === prevMessage) {
-              log.debug('--- tryForTime errored again with the same message  ...');
-            } else {
-              prevMessage = err.message;
-              log.debug('--- tryForTime error: ' + prevMessage);
-            }
-            finalMessage = err.stack || err.message;
-            return bluebird.delay(retryDelay).then(attempt);
-          });
-      }
-
-      return bluebird.try(attempt);
-    }
-
-    try(block) {
-      return this.tryForTime(config.get('timeouts.try'), block);
-    }
-
-    tryMethod(object, method, ...args) {
-      return this.try(() => object[method](...args));
-    }
-  }
-
-  return new Retry();
-}

test/common/services/retry/index.js

@@ -0,0 +1,20 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export { RetryProvider } from './retry';

test/common/services/retry/retry.js

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { retryForTruthy } from './retry_for_truthy';
+import { retryForSuccess } from './retry_for_success';
+
+export function RetryProvider({ getService }) {
+  const config = getService('config');
+  const log = getService('log');
+
+  return new class Retry {
+    async tryForTime(timeout, block) {
+      return await retryForSuccess(log, {
+        timeout,
+        methodName: 'retry.tryForTime',
+        block
+      });
+    }
+
+    async try(block) {
+      return await retryForSuccess(log, {
+        timeout: config.get('timeouts.try'),
+        methodName: 'retry.try',
+        block
+      });
+    }
+
+    async tryMethod(object, method, ...args) {
+      return await retryForSuccess(log, {
+        timeout: config.get('timeouts.try'),
+        methodName: 'retry.tryMethod',
+        block: async () => (
+          await object[method](...args)
+        )
+      });
+    }
+
+    async waitForWithTimeout(description, timeout, block) {
+      await retryForTruthy(log, {
+        timeout,
+        methodName: 'retry.waitForWithTimeout',
+        description,
+        block
+      });
+    }
+
+    async waitFor(description, block) {
+      await retryForTruthy(log, {
+        timeout: config.get('timeouts.waitFor'),
+        methodName: 'retry.waitFor',
+        description,
+        block
+      });
+    }
+  };
+}

test/common/services/retry/retry_for_success.js

@@ -0,0 +1,85 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { inspect } from 'util';
+
+const delay = ms => new Promise(resolve => (
+  setTimeout(resolve, ms)
+));
+
+const returnTrue = () => true;
+
+const defaultOnFailure = (methodName) => (lastError) => {
+  throw new Error(`${methodName} timeout: ${lastError.stack || lastError.message}`);
+};
+
+/**
+ * Run a function and return either an error or result
+ * @param {Function} block
+ */
+async function runAttempt(block) {
+  try {
+    return {
+      result: await block()
+    };
+  } catch (error) {
+    return {
+      // we rely on error being truthy and throwing falsy values is *allowed*
+      // so we cast falsy values to errors
+      error: error || new Error(`${inspect(error)} thrown`),
+    };
+  }
+}
+
+export async function retryForSuccess(log, {
+  timeout,
+  methodName,
+  block,
+  onFailure = defaultOnFailure(methodName),
+  accept = returnTrue
+}) {
+  const start = Date.now();
+  const retryDelay = 502;
+  let lastError;
+
+  while (true) {
+    if (Date.now() - start > timeout) {
+      await onFailure(lastError);
+      throw new Error('expected onFailure() option to throw an error');
+    }
+
+    const { result, error } = await runAttempt(block);
+
+    if (!error && accept(result)) {
+      return result;
+    }
+
+    if (error) {
+      if (lastError && lastError.message === error.message) {
+        log.debug(`--- ${methodName} failed again with the same message...`);
+      } else {
+        log.debug(`--- ${methodName} error: ${error.message}`);
+      }
+
+      lastError = error;
+    }
+
+    await delay(retryDelay);
+  }
+}

test/common/services/retry/retry_for_truthy.js

@@ -0,0 +1,49 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { retryForSuccess } from './retry_for_success';
+
+export async function retryForTruthy(log, {
+  timeout,
+  methodName,
+  description,
+  block
+}) {
+  log.debug(`Waiting up to ${timeout}ms for ${description}...`);
+
+  const accept = result => Boolean(result);
+
+  const onFailure = lastError => {
+    let msg = `timed out waiting for ${description}`;
+
+    if (lastError) {
+      msg = `${msg} -- last error: ${lastError.stack || lastError.message}`;
+    }
+
+    throw new Error(msg);
+  };
+
+  await retryForSuccess(log, {
+    timeout,
+    methodName,
+    block,
+    onFailure,
+    accept
+  });
+}