PSRP: Add support for reconnection retries (#49772)

* PSRP: Add support for reconnection retries

This adds support for reconnection retries to overcome sudden Connection
Refused or other recoverable states. (Either because WinRM is (re)started,
or firewall changes make it briefly unavailable)

* Fixes from review comments

lib/ansible/plugins/connection/psrp.py

@@ -94,6 +94,22 @@ options:
     vars:
     - name: ansible_psrp_connection_timeout
     default: 30
+  reconnection_retries:
+    description:
+    - The number of retries on connection errors.
+    vars:
+    - name: ansible_psrp_reconnection_retries
+    default: 0
+    version_added: '2.8'
+  reconnection_backoff:
+    description:
+    - The backoff time to use in between reconnection attempts.
+      (First sleeps X, then sleeps 2*X, then sleeps 4*X, ...)
+    - This is measured in seconds.
+    vars:
+    - name: ansible_psrp_connection_backoff
+    default: 2
+    version_added: '2.8'
   message_encryption:
     description:
     - Controls the message encryption settings, this is different from TLS
@@ -173,6 +189,7 @@ from ansible.utils.path import makedirs_safe
 HAS_PYPSRP = True
 PYPSRP_IMP_ERR = None
 try:
+    import pypsrp
     from pypsrp.complex_objects import GenericComplexObject, RunspacePoolState
     from pypsrp.exceptions import AuthenticationError, WinRMError
     from pypsrp.host import PSHost, PSHostUserInterface
@@ -504,6 +521,8 @@ if ($bytes_read -gt 0) {
         self._psrp_operation_timeout = int(self.get_option('operation_timeout'))
         self._psrp_max_envelope_size = int(self.get_option('max_envelope_size'))
         self._psrp_configuration_name = self.get_option('configuration_name')
+        self._psrp_reconnection_retries = int(self.get_option('reconnection_retries'))
+        self._psrp_reconnection_backoff = float(self.get_option('reconnection_backoff'))
 
         supported_args = []
         for auth_kwarg in AUTH_KWARGS.values():
@@ -527,6 +546,17 @@ if ($bytes_read -gt 0) {
             max_envelope_size=self._psrp_max_envelope_size,
             operation_timeout=self._psrp_operation_timeout,
         )
+
+        # Check if PSRP version supports newer reconnection_retries argument (needs pypsrp 0.3.0+)
+        if hasattr(pypsrp, 'FEATURES') and 'wsman_reconnections' in pypsrp.FEATURES:
+            self._psrp_conn_kwargs['reconnection_retries'] = self._psrp_reconnection_retries
+            self._psrp_conn_kwargs['reconnection_backoff'] = self._psrp_reconnection_backoff
+        else:
+            if self._psrp_reconnection_retries:
+                display.debug("Installed pypsrp version does not support 'reconnection_retries'.")
+            if self._psrp_reconnection_backoff:
+                display.debug("Installed pypsrp version does not support 'reconnection_backoff'.")
+
         # add in the extra args that were set
         for arg in extra_args.intersection(supported_args):
             option = self.get_option('_extras')['ansible_psrp_%s' % arg]