Merge pull request #806 from gaqzi/monit/retry-option

monit: Add retry for pending/initializing services

monitoring/monit.py

@@ -18,6 +18,7 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 #
+import time
 
 DOCUMENTATION = '''
 ---
@@ -38,6 +39,14 @@ options:
     required: true
     default: null
     choices: [ "present", "started", "stopped", "restarted", "monitored", "unmonitored", "reloaded" ]
+  timeout:
+    description:
+      - If there are pending actions for the service monitored by monit, then Ansible will check
+        for up to this many seconds to verify the the requested action has been performed.
+        Ansible will sleep for five seconds between each check.
+    required: false
+    default: 300
+    version_added: "2.0"
 requirements: [ ]
 author: "Darryl Stoflet (@dstoflet)" 
 '''
@@ -50,6 +59,7 @@ EXAMPLES = '''
 def main():
     arg_spec = dict(
         name=dict(required=True),
+        timeout=dict(default=300, type='int'),
         state=dict(required=True, choices=['present', 'started', 'restarted', 'stopped', 'monitored', 'unmonitored', 'reloaded'])
     )
 
@@ -57,17 +67,10 @@ def main():
 
     name = module.params['name']
     state = module.params['state']
+    timeout = module.params['timeout']
 
     MONIT = module.get_bin_path('monit', True)
 
-    if state == 'reloaded':
-        if module.check_mode:
-            module.exit_json(changed=True)
-        rc, out, err = module.run_command('%s reload' % MONIT)
-        if rc != 0:
-            module.fail_json(msg='monit reload failed', stdout=out, stderr=err)
-        module.exit_json(changed=True, name=name, state=state)
-    
     def status():
         """Return the status of the process in monit, or the empty string if not present."""
         rc, out, err = module.run_command('%s summary' % MONIT, check_rc=True)
@@ -86,8 +89,34 @@ def main():
         module.run_command('%s %s %s' % (MONIT, command, name), check_rc=True)
         return status()
 
-    process_status = status()
-    present = process_status != ''
+    def wait_for_monit_to_stop_pending():
+        """Fails this run if there is no status or it's pending/initalizing for timeout"""
+        timeout_time = time.time() + timeout
+        sleep_time = 5
+
+        running_status = status()
+        while running_status == '' or 'pending' in running_status or 'initializing' in running_status:
+            if time.time() >= timeout_time:
+                module.fail_json(
+                    msg='waited too long for "pending", or "initiating" status to go away ({0})'.format(
+                        running_status
+                    ),
+                    state=state
+                )
+
+            time.sleep(sleep_time)
+            running_status = status()
+
+    if state == 'reloaded':
+        if module.check_mode:
+            module.exit_json(changed=True)
+        rc, out, err = module.run_command('%s reload' % MONIT)
+        if rc != 0:
+            module.fail_json(msg='monit reload failed', stdout=out, stderr=err)
+        wait_for_monit_to_stop_pending()
+        module.exit_json(changed=True, name=name, state=state)
+
+    present = status() != ''
 
     if not present and not state == 'present':
         module.fail_json(msg='%s process not presently configured with monit' % name, name=name, state=state)
@@ -103,7 +132,8 @@ def main():
                 module.exit_json(changed=True, name=name, state=state)
         module.exit_json(changed=False, name=name, state=state)
 
-    running = 'running' in process_status 
+    wait_for_monit_to_stop_pending()
+    running = 'running' in status()
 
     if running and state in ['started', 'monitored']:
         module.exit_json(changed=False, name=name, state=state)