Add network resource modules utils function (#58273)

* Add network resource modules utils function

*  `FactsBase` parent class to handle
    facts gathering
*  `ConfigBase` parent class for resource
   module config handling
*  utils funtions for resource modules

* Fix review comments

* Fix CI issues and review comments

* Fix review comments

* Fix CI issues and minor updates

lib/ansible/module_utils/network/common/cfg/base.py

@@ -0,0 +1,18 @@
+#
+# -*- coding: utf-8 -*-
+# Copyright 2019 Red Hat
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+"""
+The base class for all resource modules
+"""
+
+from ansible.module_utils.network.common.network import get_resource_connection
+
+
+class ConfigBase(object):
+    """ The base class for all resource modules
+    """
+    def __init__(self, module):
+        self._module = module
+        self._connection = get_resource_connection(module)

lib/ansible/module_utils/network/common/facts/facts.py

@@ -0,0 +1,130 @@
+#
+# -*- coding: utf-8 -*-
+# Copyright 2019 Red Hat
+# GNU General Public License v3.0+
+# (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+"""
+The facts base class
+this contains methods common to all facts subsets
+"""
+from ansible.module_utils.network.common.network import get_resource_connection
+from ansible.module_utils.six import iteritems
+
+
+class FactsBase(object):
+    """
+    The facts base class
+    """
+    def __init__(self, module):
+        self._module = module
+        self._warnings = []
+        self._gather_subset = module.params.get('gather_subset')
+        self._gather_network_resources = module.params.get('gather_network_resources')
+        self._connection = get_resource_connection(module)
+
+        self.ansible_facts = {'ansible_network_resources': {}}
+        self.ansible_facts['ansible_gather_network_resources'] = list()
+        self.ansible_facts['ansible_net_gather_subset'] = list()
+
+        if not self._gather_subset:
+            self._gather_subset = ['!config']
+        if not self._gather_network_resources:
+            self._gather_network_resources = ['all']
+
+    def gen_runable(self, subsets, valid_subsets):
+        """ Generate the runable subset
+
+        :param module: The module instance
+        :param subsets: The provided subsets
+        :param valid_subsets: The valid subsets
+        :rtype: list
+        :returns: The runable subsets
+        """
+        runable_subsets = set()
+        exclude_subsets = set()
+        minimal_gather_subset = frozenset(['default'])
+
+        for subset in subsets:
+            if subset == 'all':
+                runable_subsets.update(valid_subsets)
+                continue
+            if subset == 'min' and minimal_gather_subset:
+                runable_subsets.update(minimal_gather_subset)
+                continue
+            if subset.startswith('!'):
+                subset = subset[1:]
+                if subset == 'min':
+                    exclude_subsets.update(minimal_gather_subset)
+                    continue
+                if subset == 'all':
+                    exclude_subsets.update(
+                        valid_subsets - minimal_gather_subset)
+                    continue
+                exclude = True
+            else:
+                exclude = False
+
+            if subset not in valid_subsets:
+                self._module.fail_json(msg='Bad subset')
+
+            if exclude:
+                exclude_subsets.add(subset)
+            else:
+                runable_subsets.add(subset)
+
+        if not runable_subsets:
+            runable_subsets.update(valid_subsets)
+        runable_subsets.difference_update(exclude_subsets)
+
+        return runable_subsets
+
+    def get_network_resources_facts(self, net_res_choices, facts_resource_obj_map, resource_facts_type=None, data=None):
+        """
+        :param net_res_choices:
+        :param fact_resource_subsets:
+        :param data: previously collected configuration
+        :return:
+        """
+        if net_res_choices:
+            if 'all' in net_res_choices:
+                net_res_choices.remove('all')
+
+        if net_res_choices:
+            if not resource_facts_type:
+                resource_facts_type = self._gather_network_resources
+
+            restorun_subsets = self.gen_runable(resource_facts_type, frozenset(net_res_choices))
+            if restorun_subsets:
+                self.ansible_facts['gather_network_resources'] = list(restorun_subsets)
+                instances = list()
+                for key in restorun_subsets:
+                    fact_cls_obj = facts_resource_obj_map.get(key)
+                    if fact_cls_obj:
+                        instances.append(fact_cls_obj(self._module))
+                    else:
+                        self._warnings.extend(["network resource fact gathering for '%s' is not supported" % key])
+
+                for inst in instances:
+                    inst.populate_facts(self._connection, self.ansible_facts, data)
+
+    def get_network_legacy_facts(self, fact_legacy_obj_map, legacy_facts_type=None):
+        if not legacy_facts_type:
+            legacy_facts_type = self._gather_subset
+
+        runable_subsets = self.gen_runable(legacy_facts_type, frozenset(fact_legacy_obj_map.keys()))
+        if runable_subsets:
+            facts = dict()
+            facts['gather_subset'] = list(runable_subsets)
+
+            instances = list()
+            for key in runable_subsets:
+                instances.append(fact_legacy_obj_map[key](self._module))
+
+            for inst in instances:
+                inst.populate()
+                facts.update(inst.facts)
+                self._warnings.extend(inst.warnings)
+
+            for key, value in iteritems(facts):
+                key = 'ansible_net_%s' % key
+                self.ansible_facts[key] = value

lib/ansible/module_utils/network/common/network.py

@@ -26,11 +26,14 @@
 # USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import traceback
+import json
 
+from ansible.module_utils._text import to_text, to_native
 from ansible.module_utils.basic import AnsibleModule
 from ansible.module_utils.basic import env_fallback
+from ansible.module_utils.connection import Connection, ConnectionError
+from ansible.module_utils.network.common.netconf import NetconfConnection
 from ansible.module_utils.network.common.parsing import Cli
-from ansible.module_utils._text import to_native
 from ansible.module_utils.six import iteritems
 
 
@@ -201,3 +204,31 @@ def register_transport(transport, default=False):
 
 def add_argument(key, value):
     NET_CONNECTION_ARGS[key] = value
+
+
+def get_resource_connection(module):
+    if hasattr(module, '_connection'):
+        return module._connection
+
+    capabilities = get_capabilities(module)
+    network_api = capabilities.get('network_api')
+    if network_api == 'cliconf':
+        module._connection = Connection(module._socket_path)
+    elif network_api == 'netconf':
+        module._connection = NetconfConnection(module._socket_path)
+    else:
+        module.fail_json(msg='Invalid connection type {0!s}'.format(network_api))
+
+    return module._connection
+
+
+def get_capabilities(module):
+    if hasattr(module, 'capabilities'):
+        return module._capabilities
+    try:
+        capabilities = Connection(module._socket_path).get_capabilities()
+    except ConnectionError as exc:
+        module.fail_json(msg=to_text(exc, errors='surrogate_then_replace'))
+    module._capabilities = json.loads(capabilities)
+
+    return module._capabilities

lib/ansible/module_utils/network/common/utils.py

@@ -32,14 +32,16 @@ import re
 import ast
 import operator
 import socket
+import json
 
 from itertools import chain
 from socket import inet_aton
+from json import dumps
 
-from ansible.module_utils._text import to_text
+from ansible.module_utils._text import to_text, to_bytes
 from ansible.module_utils.common._collections_compat import Mapping
 from ansible.module_utils.six import iteritems, string_types
-from ansible.module_utils.basic import AnsibleFallbackNotFound
+from ansible.module_utils import basic
 from ansible.module_utils.parsing.convert_bool import boolean
 
 # Backwards compatibility for 3rd party modules
@@ -195,7 +197,7 @@ class Entity(object):
                             fallback_args = item
                     try:
                         value[name] = fallback_strategy(*fallback_args, **fallback_kwargs)
-                    except AnsibleFallbackNotFound:
+                    except basic.AnsibleFallbackNotFound:
                         continue
 
             if attr.get('required') and value.get(name) is None:
@@ -438,10 +440,140 @@ def _fallback(fallback):
             args = item
     try:
         return strategy(*args, **kwargs)
-    except AnsibleFallbackNotFound:
+    except basic.AnsibleFallbackNotFound:
         pass
 
 
+def generate_dict(spec):
+    """
+    Generate dictionary which is in sync with argspec
+
+    :param spec: A dictionary that is the argspec of the module
+    :rtype: A dictionary
+    :returns: A dictionary in sync with argspec with default value
+    """
+    obj = {}
+    if not spec:
+        return obj
+
+    for key, val in iteritems(spec):
+        if 'default' in val:
+            dct = {key: val['default']}
+        elif 'type' in val and val['type'] == 'dict':
+            dct = {key: generate_dict(val['options'])}
+        else:
+            dct = {key: None}
+        obj.update(dct)
+    return obj
+
+
+def parse_conf_arg(cfg, arg):
+    """
+    Parse config based on argument
+
+    :param cfg: A text string which is a line of configuration.
+    :param arg: A text string which is to be matched.
+    :rtype: A text string
+    :returns: A text string if match is found
+    """
+    match = re.search(r'%s (.+)(\n|$)' % arg, cfg, re.M)
+    if match:
+        result = match.group(1).strip()
+    else:
+        result = None
+    return result
+
+
+def parse_conf_cmd_arg(cfg, cmd, res1, res2=None, delete_str='no'):
+    """
+    Parse config based on command
+
+    :param cfg: A text string which is a line of configuration.
+    :param cmd: A text string which is the command to be matched
+    :param res1: A text string to be returned if the command is present
+    :param res2: A text string to be returned if the negate command
+                 is present
+    :param delete_str: A text string to identify the start of the
+                 negate command
+    :rtype: A text string
+    :returns: A text string if match is found
+    """
+    match = re.search(r'\n\s+%s(\n|$)' % cmd, cfg)
+    if match:
+        return res1
+    if res2 is not None:
+        match = re.search(r'\n\s+%s %s(\n|$)' % (delete_str, cmd), cfg)
+        if match:
+            return res2
+    return None
+
+
+def get_xml_conf_arg(cfg, path, data='text'):
+    """
+    :param cfg: The top level configuration lxml Element tree object
+    :param path: The relative xpath w.r.t to top level element (cfg)
+           to be searched in the xml hierarchy
+    :param data: The type of data to be returned for the matched xml node.
+        Valid values are text, tag, attrib, with default as text.
+    :return: Returns the required type for the matched xml node or else None
+    """
+    match = cfg.xpath(path)
+    if len(match):
+        if data == 'tag':
+            result = getattr(match[0], 'tag')
+        elif data == 'attrib':
+            result = getattr(match[0], 'attrib')
+        else:
+            result = getattr(match[0], 'text')
+    else:
+        result = None
+    return result
+
+
+def remove_empties(cfg_dict):
+    """
+    Generate final config dictionary
+
+    :param cfg_dict: A dictionary parsed in the facts system
+    :rtype: A dictionary
+    :returns: A dictionary by eliminating keys that have null values
+    """
+    final_cfg = {}
+    if not cfg_dict:
+        return final_cfg
+
+    for key, val in iteritems(cfg_dict):
+        dct = None
+        if isinstance(val, dict):
+            child_val = remove_empties(val)
+            if child_val:
+                dct = {key: child_val}
+        elif (isinstance(val, list) and val
+              and all([isinstance(x, dict) for x in val])):
+            child_val = [remove_empties(x) for x in val]
+            if child_val:
+                dct = {key: child_val}
+        elif val not in [None, [], {}, (), '']:
+            dct = {key: val}
+        if dct:
+            final_cfg.update(dct)
+    return final_cfg
+
+
+def validate_config(spec, data):
+    """
+    Validate if the input data against the AnsibleModule spec format
+    :param spec: Ansible argument spec
+    :param data: Data to be validated
+    :return:
+    """
+    params = basic._ANSIBLE_ARGS
+    basic._ANSIBLE_ARGS = to_bytes(json.dumps({'ANSIBLE_MODULE_ARGS': data}))
+    validated_data = basic.AnsibleModule(spec).params
+    basic._ANSIBLE_ARGS = params
+    return validated_data
+
+
 class Template:
 
     def __init__(self):