fix missing attribs with dirct module execution (#53875)

* fix missing attribs with dirct module execution
* also make remote tmp handling smarter
 update tests
* set default if attrib does not exist
* add simple test

changelogs/fragments/always_module_attribs.yml

@@ -0,0 +1,2 @@
+bugfixes:
+    - ensure we always have internal module attributes set, even if not being passed (fixes using modules as script)

lib/ansible/module_utils/basic.py

@@ -27,8 +27,6 @@ FILE_ATTRIBUTES = {
     'Z': 'compresseddirty',
 }
 
-PASS_BOOLS = ('no_log', 'debug', 'diff')
-
 # Ansible modules can be written in any language.
 # The functions available here can be used to do many common tasks,
 # to simplify development of Python modules.
@@ -157,6 +155,7 @@ from ansible.module_utils.common.parameters import (
     list_deprecations,
     list_no_log_values,
     PASS_VARS,
+    PASS_BOOLS,
 )
 
 from ansible.module_utils.six import (
@@ -711,8 +710,10 @@ class AnsibleModule(object):
         if self._tmpdir is None:
             basedir = None
 
-            basedir = os.path.expanduser(os.path.expandvars(self._remote_tmp))
+            if self._remote_tmp is not None:
-            if not os.path.exists(basedir):
+                basedir = os.path.expanduser(os.path.expandvars(self._remote_tmp))
+
+            if basedir is not None and not os.path.exists(basedir):
                 try:
                     os.makedirs(basedir, mode=0o700)
                 except (OSError, IOError) as e:
@@ -1441,20 +1442,27 @@ class AnsibleModule(object):
         if legal_inputs is None:
             legal_inputs = self._legal_inputs
 
-        for (k, v) in list(param.items()):
+        for k in list(param.keys()):
 
             if check_invalid_arguments and k not in legal_inputs:
                 unsupported_parameters.add(k)
-            elif k.startswith('_ansible_'):
-                # handle setting internal properties from internal ansible vars
-                key = k.replace('_ansible_', '')
-                if key in PASS_BOOLS:
-                    setattr(self, PASS_VARS[key], self.boolean(v))
-                else:
-                    setattr(self, PASS_VARS[key], v)
 
-                # clean up internal params:
+        for k in PASS_VARS:
-                del self.params[k]
+            # handle setting internal properties from internal ansible vars
+            param_key = '_ansible_%s' % k
+            if param_key in param:
+                if k in PASS_BOOLS:
+                    setattr(self, PASS_VARS[k][0], self.boolean(param[param_key]))
+                else:
+                    setattr(self, PASS_VARS[k][0], param[param_key])
+
+                # clean up internal top level params:
+                if param_key in self.params:
+                    del self.params[param_key]
+            else:
+                # use defaults if not already set
+                if not hasattr(self, PASS_VARS[k][0]):
+                    setattr(self, PASS_VARS[k][0], PASS_VARS[k][1])
 
         if unsupported_parameters:
             msg = "Unsupported parameters for (%s) module: %s" % (self._name, ', '.join(sorted(list(unsupported_parameters))))

lib/ansible/module_utils/common/parameters.py

@@ -18,24 +18,28 @@ from ansible.module_utils.six import (
 # Python2 & 3 way to get NoneType
 NoneType = type(None)
 
+# if adding boolean attribute, also add to PASS_BOOL
+# some of this dupes defaults from controller config
 PASS_VARS = {
-    'check_mode': 'check_mode',
+    'check_mode': ('check_mode', False),
-    'debug': '_debug',
+    'debug': ('_debug', False),
-    'diff': '_diff',
+    'diff': ('_diff', False),
-    'keep_remote_files': '_keep_remote_files',
+    'keep_remote_files': ('_keep_remote_files', False),
-    'module_name': '_name',
+    'module_name': ('_name', None),
-    'no_log': 'no_log',
+    'no_log': ('no_log', False),
-    'remote_tmp': '_remote_tmp',
+    'remote_tmp': ('_remote_tmp', None),
-    'selinux_special_fs': '_selinux_special_fs',
+    'selinux_special_fs': ('_selinux_special_fs', ['fuse', 'nfs', 'vboxsf', 'ramfs', '9p']),
-    'shell_executable': '_shell',
+    'shell_executable': ('_shell', '/bin/sh'),
-    'socket': '_socket_path',
+    'socket': ('_socket_path', None),
-    'string_conversion_action': '_string_conversion_action',
+    'string_conversion_action': ('_string_conversion_action', 'warn'),
-    'syslog_facility': '_syslog_facility',
+    'syslog_facility': ('_syslog_facility', 'INFO'),
-    'tmpdir': '_tmpdir',
+    'tmpdir': ('_tmpdir', None),
-    'verbosity': '_verbosity',
+    'verbosity': ('_verbosity', 0),
-    'version': 'ansible_version',
+    'version': ('ansible_version', '0.0'),
 }
 
+PASS_BOOLS = ('check_mode', 'debug', 'diff', 'keep_remote_files', 'no_log')
+
 
 def _return_datastructure_name(obj):
     """ Return native stringified values from datastructures.

test/integration/targets/run_modules/aliases

@@ -0,0 +1 @@
+shippable/posix/group3

test/integration/targets/run_modules/args.json

@@ -0,0 +1 @@
+{ "ANSIBLE_MODULE_ARGS": {} }

test/integration/targets/run_modules/library/test.py

@@ -0,0 +1,7 @@
+#!/usr/bin/python
+
+from ansible.module_utils.basic import AnsibleModule
+
+module = AnsibleModule(argument_spec=dict())
+
+module.exit_json(**{'tempdir': module._remote_tmp})

test/integration/targets/run_modules/runme.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# test running module directly
+python.py library/test.py args.json

test/units/modules/conftest.py

@@ -20,7 +20,7 @@ def patch_ansible_module(request, mocker):
         if '_ansible_remote_tmp' not in request.param['ANSIBLE_MODULE_ARGS']:
             request.param['ANSIBLE_MODULE_ARGS']['_ansible_remote_tmp'] = '/tmp'
         if '_ansible_keep_remote_files' not in request.param['ANSIBLE_MODULE_ARGS']:
-            request.param['ANSIBLE_MODULE_ARGS']['_ansible_keep_remote_files'] = '/tmp'
+            request.param['ANSIBLE_MODULE_ARGS']['_ansible_keep_remote_files'] = False
         args = json.dumps(request.param)
     else:
         raise Exception('Malformed data to the patch_ansible_module pytest fixture')