First swing at making module_common.py more v2-ish

v2/ansible/executor/module_common.py

@@ -1,4 +1,5 @@
 # (c) 2013-2014, Michael DeHaan <michael.dehaan@gmail.com>
+# (c) 2015 Toshio Kuratomi <tkuratomi@ansible.com>
 #
 # This file is part of Ansible
 #
@@ -15,6 +16,10 @@
 # You should have received a copy of the GNU General Public License
 # along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
 
+# Make coding more python3-ish
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
 # from python and deps
 from cStringIO import StringIO
 import inspect
@@ -34,151 +39,153 @@ REPLACER_COMPLEX = "\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
 REPLACER_WINDOWS = "# POWERSHELL_COMMON"
 REPLACER_VERSION = "\"<<ANSIBLE_VERSION>>\""
 
-class ModuleReplacer(object):
+# We could end up writing out parameters with unicode characters so we need to
+# specify an encoding for the python source file
+ENCODING_STRING = '# -*- coding: utf-8 -*-'
 
+# TODO: Is there a reason we don't use __file__ here?
+#       Will this fail if ansible is run from an egg/wheel?  Do we care?
+_THIS_FILE = inspect.getfile(inspect.currentframe())
+# we've moved the module_common relative to the snippets, so fix the path
+_SNIPPET_PATH = os.path.join(os.path.dirname(this_file), '..', 'module_utils')
+
+# ******************************************************************************
+
+def _slurp(path):
+    if not os.path.exists(path):
+        raise AnsibleError("imported module support code does not exist at %s" % path)
+    fd = open(path)
+    data = fd.read()
+    fd.close()
+    return data
+
+def _find_snippet_imports(module_data, module_path, strip_comments):
     """
-    The Replacer is used to insert chunks of code into modules before
+    Given the source of the module, convert it to a Jinja2 template to insert
-    transfer.  Rather than doing classical python imports, this allows for more
+    module code and return whether it's a new or old style module.
-    efficient transfer in a no-bootstrapping scenario by not moving extra files
+    """
-    over the wire, and also takes care of embedding arguments in the transferred
+
-    modules.  
+    module_style = 'old'
+    if REPLACER in module_data:
+        module_style = 'new'
+    elif 'from ansible.module_utils.' in module_data:
+        module_style = 'new'
+    elif 'WANT_JSON' in module_data:
+        module_style = 'non_native_want_json'
+
+    output = StringIO()
+    lines = module_data.split('\n')
+    snippet_names = []
+
+    for line in lines:
+
+        if REPLACER in line:
+            output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
+            snippet_names.append('basic')
+        if REPLACER_WINDOWS in line:
+            ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
+            output.write(ps_data)
+            snippet_names.append('powershell')
+        elif line.startswith('from ansible.module_utils.'):
+            tokens=line.split(".")
+            import_error = False
+            if len(tokens) != 3:
+                import_error = True
+            if " import *" not in line:
+                import_error = True
+            if import_error:
+                raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'" % module_path)
+            snippet_name = tokens[2].split()[0]
+            snippet_names.append(snippet_name)
+            output.write(_slurp(os.path.join(_SNIPPET_PATH, snippet_name + ".py")))
+        else:
+            if strip_comments and line.startswith("#") or line == '':
+                pass
+            output.write(line)
+            output.write("\n")
+
+    if not module_path.endswith(".ps1"):
+        # Unixy modules
+        if len(snippet_names) > 0 and not 'basic' in snippet_names:
+            raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path)
+    else:
+        # Windows modules
+        if len(snippet_names) > 0 and not 'powershell' in snippet_names:
+            raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
+
+    return (output.getvalue(), module_style)
+
+# ******************************************************************************
+
+def modify_module(module_path, module_args, strip_comments=False):
+    """
+    Used to insert chunks of code into modules before transfer rather than
+    doing regular python imports.  This allows for more efficient transfer in
+    a non-bootstrapping scenario by not moving extra files over the wire and
+    also takes care of embedding arguments in the transferred modules.
 
     This version is done in such a way that local imports can still be
     used in the module code, so IDEs don't have to be aware of what is going on.
 
     Example:
 
-    from ansible.module_utils.basic import * 
+    from ansible.module_utils.basic import *
 
-       ... will result in the insertion basic.py into the module
+       ... will result in the insertion of basic.py into the module
-
+       from the module_utils/ directory in the source tree.
-    from the module_utils/ directory in the source tree.
 
     All modules are required to import at least basic, though there will also
     be other snippets.
 
+    For powershell, there's equivalent conventions like this:
+
     # POWERSHELL_COMMON
 
-    Also results in the inclusion of the common code in powershell.ps1
+    which results in the inclusion of the common code from powershell.ps1
 
     """
 
-    # ******************************************************************************
+    with open(module_path) as f:
 
-    def __init__(self, strip_comments=False):
+        # read in the module source
-        # FIXME: these members need to be prefixed with '_' and the rest of the file fixed
+        module_data = f.read()
-        this_file = inspect.getfile(inspect.currentframe())
-        # we've moved the module_common relative to the snippets, so fix the path
-        self.snippet_path = os.path.join(os.path.dirname(this_file), '..', 'module_utils')
-        self.strip_comments = strip_comments
 
-    # ******************************************************************************
+        (module_data, module_style) = _find_snippet_imports(module_data, module_path, strip_comments)
 
+        #module_args_json = jsonify(module_args)
+        module_args_json = json.dumps(module_args)
+        encoded_args = repr(module_args_json.encode('utf-8'))
 
-    def slurp(self, path):
+        # these strings should be part of the 'basic' snippet which is required to be included
-        if not os.path.exists(path):
+        module_data = module_data.replace(REPLACER_VERSION, repr(__version__))
-            raise AnsibleError("imported module support code does not exist at %s" % path)
+        module_data = module_data.replace(REPLACER_COMPLEX, encoded_args)
-        fd = open(path)
-        data = fd.read()
-        fd.close()
-        return data
 
-    def _find_snippet_imports(self, module_data, module_path):
+        # FIXME: we're not passing around an inject dictionary anymore, so
-        """
+        #        this needs to be fixed with whatever method we use for vars
-        Given the source of the module, convert it to a Jinja2 template to insert
+        #        like this moving forward
-        module code and return whether it's a new or old style module.
+        #if module_style == 'new':
-        """
+        #    facility = C.DEFAULT_SYSLOG_FACILITY
+        #    if 'ansible_syslog_facility' in inject:
+        #        facility = inject['ansible_syslog_facility']
+        #    module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility)
 
-        module_style = 'old'
+        lines = module_data.split("\n", 1)
-        if REPLACER in module_data:
+        shebang = None
-            module_style = 'new'
+        if lines[0].startswith("#!"):
-        elif 'from ansible.module_utils.' in module_data:
+            shebang = lines[0].strip()
-            module_style = 'new'
+            args = shlex.split(str(shebang[2:]))
-        elif 'WANT_JSON' in module_data:
+            interpreter = args[0]
-            module_style = 'non_native_want_json'
+            interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
-      
-        output = StringIO()
-        lines = module_data.split('\n')
-        snippet_names = []
 
-        for line in lines:
+            # FIXME: more inject stuff here...
+            #if interpreter_config in inject:
+            #    lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:]))
 
-            if REPLACER in line:
+            lines.insert(1, ENCODING_STRING)
-                output.write(self.slurp(os.path.join(self.snippet_path, "basic.py")))
-                snippet_names.append('basic')
-            if REPLACER_WINDOWS in line:
-                ps_data = self.slurp(os.path.join(self.snippet_path, "powershell.ps1"))
-                output.write(ps_data)
-                snippet_names.append('powershell')
-            elif line.startswith('from ansible.module_utils.'):
-                tokens=line.split(".")
-                import_error = False
-                if len(tokens) != 3:
-                    import_error = True
-                if " import *" not in line:
-                    import_error = True
-                if import_error:
-                    raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.basic import *'" % module_path)
-                snippet_name = tokens[2].split()[0]
-                snippet_names.append(snippet_name)
-                output.write(self.slurp(os.path.join(self.snippet_path, snippet_name + ".py")))
-            else:
-                if self.strip_comments and line.startswith("#") or line == '':
-                    pass
-                output.write(line)
-                output.write("\n")
-
-        if not module_path.endswith(".ps1"):
-            # Unixy modules
-            if len(snippet_names) > 0 and not 'basic' in snippet_names:
-                raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path) 
         else:
-            # Windows modules
+            lines.insert(0, ENCODING_STRING)
-            if len(snippet_names) > 0 and not 'powershell' in snippet_names:
-                raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path) 
 
-        return (output.getvalue(), module_style)
+        module_data = "\n".join(lines)
 
-    # ******************************************************************************
+        return (module_data, module_style, shebang)
-
-    def modify_module(self, module_path, module_args):
-
-        with open(module_path) as f:
-
-            # read in the module source
-            module_data = f.read()
-
-            (module_data, module_style) = self._find_snippet_imports(module_data, module_path)
-
-            #module_args_json = jsonify(module_args)
-            module_args_json = json.dumps(module_args)
-            encoded_args = repr(module_args_json.encode('utf-8'))
-
-            # these strings should be part of the 'basic' snippet which is required to be included
-            module_data = module_data.replace(REPLACER_VERSION, repr(__version__))
-            module_data = module_data.replace(REPLACER_COMPLEX, encoded_args)
-
-            # FIXME: we're not passing around an inject dictionary anymore, so
-            #        this needs to be fixed with whatever method we use for vars
-            #        like this moving forward
-            #if module_style == 'new':
-            #    facility = C.DEFAULT_SYSLOG_FACILITY
-            #    if 'ansible_syslog_facility' in inject:
-            #        facility = inject['ansible_syslog_facility']
-            #    module_data = module_data.replace('syslog.LOG_USER', "syslog.%s" % facility)
-
-            lines = module_data.split("\n")
-            shebang = None
-            if lines[0].startswith("#!"):
-                shebang = lines[0].strip()
-                args = shlex.split(str(shebang[2:]))
-                interpreter = args[0]
-                interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
-
-                # FIXME: more inject stuff here...
-                #if interpreter_config in inject:
-                #    lines[0] = shebang = "#!%s %s" % (inject[interpreter_config], " ".join(args[1:]))
-                #    module_data = "\n".join(lines)
-
-            return (module_data, module_style, shebang)
 

v2/ansible/module_utils/basic.py

@@ -43,7 +43,7 @@ BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE
 # can be inserted in any module source automatically by including
 # #<<INCLUDE_ANSIBLE_MODULE_COMMON>> on a blank line by itself inside
 # of an ansible module. The source of this common code lives
-# in lib/ansible/module_common.py
+# in ansible/executor/module_common.py
 
 import locale
 import os

v2/ansible/modules/core

@@ -1 +1 @@
-Subproject commit 34784b7a617aa35d3b994c9f0795567afc6fb0b0
+Subproject commit 095f8681dbdfd2e9247446822e953287c9bca66c

v2/ansible/plugins/action/__init__.py

@@ -29,7 +29,7 @@ import time
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
-from ansible.executor.module_common import ModuleReplacer
+from ansible.executor.module_common import modify_module
 from ansible.parsing.utils.jsonify import jsonify
 from ansible.plugins import shell_loader
 
@@ -78,7 +78,7 @@ class ActionBase:
     def _configure_module(self, module_name, module_args):
         '''
         Handles the loading and templating of the module code through the
-        ModuleReplacer class.
+        modify_module() function.
         '''
 
         # Search module path(s) for named module.
@@ -94,7 +94,7 @@ class ActionBase:
                                    "run 'git submodule update --init --recursive' to correct this problem." % (module_name))
 
         # insert shared code and arguments into the module
-        (module_data, module_style, module_shebang) = ModuleReplacer().modify_module(module_path, module_args)
+        (module_data, module_style, module_shebang) = modify_module(module_path, module_args)
 
         return (module_style, module_shebang, module_data)
 

v2/hacking/test-module

@@ -34,8 +34,9 @@ import os
 import subprocess
 import traceback
 import optparse
-import ansible.utils as utils
+
-import ansible.module_common as module_common
+from ansible import utils
+from ansible import module_common
 import ansible.constants as C
 
 try:
@@ -87,8 +88,6 @@ def boilerplate_module(modfile, args, interpreter, check):
     #module_data = module_fh.read()
     #module_fh.close()
 
-    replacer = module_common.ModuleReplacer()
-
     #included_boilerplate = module_data.find(module_common.REPLACER) != -1 or module_data.find("import ansible.module_utils") != -1
 
     complex_args = {}
@@ -116,7 +115,7 @@ def boilerplate_module(modfile, args, interpreter, check):
     if check:
          complex_args['CHECKMODE'] = True
 
-    (module_data, module_style, shebang) = replacer.modify_module(
+    (module_data, module_style, shebang) = module_common.modify_module(
         modfile, 
         complex_args,
         args,