From b571ecdfec041c86ba0ebdba3f2964932b653329 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Sun, 10 Apr 2016 12:33:16 -0700 Subject: [PATCH] Move module arg passing from the environment to stdin (from the wrapper to the module) --- lib/ansible/executor/module_common.py | 126 ++++++++++++++++---------- lib/ansible/module_utils/basic.py | 73 +++++++-------- 2 files changed, 111 insertions(+), 88 deletions(-) diff --git a/lib/ansible/executor/module_common.py b/lib/ansible/executor/module_common.py index 144af8c1003..c06f5fa20aa 100644 --- a/lib/ansible/executor/module_common.py +++ b/lib/ansible/executor/module_common.py @@ -97,9 +97,35 @@ if sys.version_info < (3,): else: unicode = str +try: + # Python-2.6+ + from io import BytesIO as IOStream +except ImportError: + # Python < 2.6 + from StringIO import StringIO as IOStream + ZIPDATA = """%(zipdata)s""" -def debug(command, zipped_mod): +def invoke_module(module_path, json_params): + pythonpath = os.environ.get('PYTHONPATH') + if pythonpath: + os.environ['PYTHONPATH'] = ':'.join((module_path, pythonpath)) + else: + os.environ['PYTHONPATH'] = module_path + + p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + (stdout, stderr) = p.communicate(json_params) + + if not isinstance(stderr, (bytes, unicode)): + stderr = stderr.read() + if not isinstance(stdout, (bytes, unicode)): + stdout = stdout.read() + sys.stderr.write(stderr) + sys.stdout.write(stdout) + + return p.returncode + +def debug(command, zipped_mod, json_params): # The code here normally doesn't run. It's only used for debugging on the # remote machine. Run with ANSIBLE_KEEP_REMOTE_FILES=1 envvar and -vvv # to save the module file remotely. Login to the remote machine and use @@ -107,7 +133,7 @@ def debug(command, zipped_mod): # files. Edit the source files to instrument the code or experiment with # different values. Then use /path/to/module execute to run the extracted # files you've edited instead of the actual zipped module. - # + # Okay to use __file__ here because we're running from a kept file basedir = os.path.dirname(__file__) if command == 'explode': @@ -120,6 +146,7 @@ def debug(command, zipped_mod): for filename in z.namelist(): if filename.startswith('/'): raise Exception('Something wrong with this module zip file: should not contain absolute paths') + dest_filename = os.path.join(basedir, filename) if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename): os.makedirs(dest_filename) @@ -130,26 +157,17 @@ def debug(command, zipped_mod): f = open(dest_filename, 'w') f.write(z.read(filename)) f.close() + print('Module expanded into:') print('%%s' %% os.path.join(basedir, 'ansible')) + exitcode = 0 + elif command == 'execute': # Execute the exploded code instead of executing the module from the # embedded ZIPDATA. This allows people to easily run their modified # code on the remote machine to see how changes will affect it. - pythonpath = os.environ.get('PYTHONPATH') - if pythonpath: - os.environ['PYTHONPATH'] = ':'.join((basedir, pythonpath)) - else: - os.environ['PYTHONPATH'] = basedir - p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if not isinstance(stderr, (bytes, unicode)): - stderr = stderr.read() - if not isinstance(stdout, (bytes, unicode)): - stdout = stdout.read() - sys.stderr.write(stderr) - sys.stdout.write(stdout) - sys.exit(p.returncode) + exitcode = invoke_module(basedir, json_params) + elif command == 'excommunicate': # This attempts to run the module in-process (by importing a main # function and then calling it). It is not the way ansible generally @@ -159,41 +177,41 @@ def debug(command, zipped_mod): # when using this that are only artifacts of how we're invoking here, # not actual bugs (as they don't affect the real way that we invoke # ansible modules) + sys.stdin = IOStream(json_params) sys.path.insert(0, basedir) from ansible.module_exec.%(ansible_module)s.__main__ import main main() - -os.environ['ANSIBLE_MODULE_ARGS'] = %(args)s -os.environ['ANSIBLE_MODULE_CONSTANTS'] = %(constants)s - -try: - temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_') - os.write(temp_fd, base64.b64decode(ZIPDATA)) - if len(sys.argv) == 2: - debug(sys.argv[1], temp_path) + print('WARNING: Module returned to wrapper instead of exiting') + sys.exit(1) else: - pythonpath = os.environ.get('PYTHONPATH') - if pythonpath: - os.environ['PYTHONPATH'] = ':'.join((temp_path, pythonpath)) - else: - os.environ['PYTHONPATH'] = temp_path - p = subprocess.Popen(['%(interpreter)s', '-m', 'ansible.module_exec.%(ansible_module)s.__main__'], env=os.environ, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - (stdout, stderr) = p.communicate() - if not isinstance(stderr, (bytes, unicode)): - stderr = stderr.read() - if not isinstance(stdout, (bytes, unicode)): - stdout = stdout.read() - sys.stderr.write(stderr) - sys.stdout.write(stdout) - sys.exit(p.returncode) + print('WARNING: Unknown debug command. Doing nothing.') + exitcode = 0 + + return exitcode + +if __name__ == '__main__': + ZIPLOADER_PARAMS = %(params)s -finally: try: + temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_') + os.write(temp_fd, base64.b64decode(ZIPDATA)) os.close(temp_fd) - os.remove(temp_path) - except NameError: - # mkstemp failed - pass + if len(sys.argv) == 2: + exitcode = debug(sys.argv[1], temp_path, ZIPLOADER_PARAMS) + else: + exitcode = invoke_module(temp_path, ZIPLOADER_PARAMS) + finally: + try: + try: + os.close(temp_fd) + except OSError: + # Already closed + pass + os.remove(temp_path) + except NameError: + # mkstemp failed + pass + sys.exit(exitcode) ''' class ModuleDepFinder(ast.NodeVisitor): @@ -336,19 +354,21 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta if module_style in ('old', 'non_native_want_json'): return module_data, module_style, shebang - module_args_json = to_bytes(json.dumps(module_args)) - output = BytesIO() snippet_names = set() if module_substyle == 'python': # ziploader for new-style python classes - python_repred_args = to_bytes(repr(module_args_json)) constants = dict( SELINUX_SPECIAL_FS=C.DEFAULT_SELINUX_SPECIAL_FS, SYSLOG_FACILITY=_get_facility(task_vars), ) - python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict') + params = dict(ANSIBLE_MODULE_ARGS=module_args, + ANSIBLE_MODULE_CONSTANTS=constants, + ) + #python_repred_args = to_bytes(repr(module_args_json)) + #python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict') + python_repred_params = to_bytes(repr(json.dumps(params)), errors='strict') try: compression_method = getattr(zipfile, module_compression) @@ -411,8 +431,9 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta output.write(to_bytes(STRIPPED_ZIPLOADER_TEMPLATE % dict( zipdata=zipdata, ansible_module=module_name, - args=python_repred_args, - constants=python_repred_constants, + #args=python_repred_args, + #constants=python_repred_constants, + params=python_repred_params, shebang=shebang, interpreter=interpreter, coding=ENCODING_STRING, @@ -437,6 +458,8 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta continue output.write(line + b'\n') module_data = output.getvalue() + + module_args_json = to_bytes(json.dumps(module_args)) module_data = module_data.replace(REPLACER_JSONARGS, module_args_json) # Sanity check from 1.x days. This is currently useless as we only @@ -447,11 +470,14 @@ def _find_snippet_imports(module_name, module_data, module_path, module_args, ta raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path) elif module_substyle == 'jsonargs': + module_args_json = to_bytes(json.dumps(module_args)) + # these strings could be included in a third-party module but # officially they were included in the 'basic' snippet for new-style # python modules (which has been replaced with something else in # ziploader) If we remove them from jsonargs-style module replacer # then we can remove them everywhere. + python_repred_args = to_bytes(repr(module_args_json)) module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__))) module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args) module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS))) diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index f202dba0282..a06814a4c56 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -223,23 +223,6 @@ from ansible import __version__ # Backwards compat. New code should just import and use __version__ ANSIBLE_VERSION = __version__ -try: - # MODULE_COMPLEX_ARGS is an old name kept for backwards compat - MODULE_COMPLEX_ARGS = os.environ.pop('ANSIBLE_MODULE_ARGS') -except KeyError: - # This file might be used for its utility functions. So don't fail if - # running outside of a module environment (will fail in _load_params() - # instead) - MODULE_COMPLEX_ARGS = None - -try: - # ARGS are for parameters given in the playbook. Constants are for things - # that ansible needs to configure controller side but are passed to all - # modules. - MODULE_CONSTANTS = os.environ.pop('ANSIBLE_MODULE_CONSTANTS') -except KeyError: - MODULE_CONSTANTS = None - FILE_COMMON_ARGUMENTS=dict( src = dict(), mode = dict(type='raw'), @@ -560,7 +543,6 @@ class AnsibleModule(object): if k not in self.argument_spec: self.argument_spec[k] = v - self._load_constants() self._load_params() self._set_fallbacks() @@ -1452,32 +1434,47 @@ class AnsibleModule(object): continue def _load_params(self): - ''' read the input and set the params attribute''' - if MODULE_COMPLEX_ARGS is None: + ''' read the input and set the params attribute. Sets the constants as well.''' + buffer = sys.stdin.read() + try: + params = json.loads(buffer) + except ValueError: # This helper used too early for fail_json to work. - print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment. Unable to figure out what parameters were passed", "failed": true}') + print('{"msg": "Error: Module unable to decode valid JSON on stdin. Unable to figure out what parameters were passed", "failed": true}') sys.exit(1) - params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS)) - if params is None: - params = dict() - self.params = params - - def _load_constants(self): - ''' read the input and set the constants attribute''' - if MODULE_CONSTANTS is None: + try: + self.params = params['ANSIBLE_MODULE_ARGS'] + self.constants = params['ANSIBLE_MODULE_CONSTANTS'] + except KeyError: # This helper used too early for fail_json to work. - print('{"msg": "Error: ANSIBLE_MODULE_CONSTANTS not found in environment. Unable to figure out what constants were passed", "failed": true}') + print('{"msg": "Error: Module unable to locate ANSIBLE_MODULE_ARGS and ANSIBLE_MODULE_CONSTANTS in json data from stdin. Unable to figure out what parameters were passed", "failed": true}') sys.exit(1) - # Make constants into "native string" - if sys.version_info >= (3,): - constants = json_dict_bytes_to_unicode(json.loads(MODULE_CONSTANTS)) - else: - constants = json_dict_unicode_to_bytes(json.loads(MODULE_CONSTANTS)) - if constants is None: - constants = dict() - self.constants = constants +# import select +# buffer = '' +# while True: +# input_list = select.select([sys.stdin], [], [], 5.0)[0] +# if sys.stdin not in input_list: +# # This helper used too early for fail_json to work. +# print('{"msg": "Error: Module unable to read arguments from stdin. Unable to figure out what parameters were passed", "failed": true}') +# sys.exit(1) +# buffer += sys.stdin.read() +# if json.loads(buffer): +# +# for line in sys.stdin: +# if line is None: +# print('s') +# data = sys.stdin.read() +# if MODULE_COMPLEX_ARGS is None: +# # This helper used too early for fail_json to work. +# print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment. Unable to figure out what parameters were passed", "failed": true}') +# sys.exit(1) +# +# params = json_dict_unicode_to_bytes(json.loads(data)) +# if params is None: +# params = dict() +# self.params = params def _log_to_syslog(self, msg): if HAS_SYSLOG: