diff --git a/CHANGELOG.md b/CHANGELOG.md index 4faa8f2ed3b..09023135699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,7 +58,14 @@ Some other notable changes: * ec2_ami_search: support for SSD and IOPS provisioned EBS images * can set ansible_sudo_exe as an inventory variable which allows specifying a different sudo (or equivalent) command -* git module: Submodule handling has changed. Previously if you used the ``recursive`` parameter to handle submodules, ansible would track the submodule upstream's head revision. This has been changed to checkout the version of the submodule specified in the superproject's git repository. This is inline with what git submodule update does. If you want the old behaviour use the new module parameter track_submodules=yes +* git module: Submodule handling has changed. Previously if you used the + ``recursive`` parameter to handle submodules, ansible would track the + submodule upstream's head revision. This has been changed to checkout the + version of the submodule specified in the superproject's git repository. + This is inline with what git submodule update does. If you want the old + behaviour use the new module parameter track_submodules=yes +* Checksumming of transferred files has been made more portable and now uses + the sha1 algorithm instead of md5 to be compatible with FIPS-140. And various other bug fixes and improvements ... diff --git a/docsite/rst/developing_modules.rst b/docsite/rst/developing_modules.rst index 4a331626db1..aff5fab5567 100644 --- a/docsite/rst/developing_modules.rst +++ b/docsite/rst/developing_modules.rst @@ -262,7 +262,7 @@ And failures are just as simple (where 'msg' is a required parameter to explain module.fail_json(msg="Something fatal happened") -There are also other useful functions in the module class, such as module.md5(path). See +There are also other useful functions in the module class, such as module.sha1(path). See lib/ansible/module_common.py in the source checkout for implementation details. Again, modules developed this way are best tested with the hacking/test-module script in the git diff --git a/docsite/rst/playbooks_prompts.rst b/docsite/rst/playbooks_prompts.rst index c20e59e0791..29fc218fe86 100644 --- a/docsite/rst/playbooks_prompts.rst +++ b/docsite/rst/playbooks_prompts.rst @@ -55,7 +55,7 @@ entered value so you can use it, for instance, with the user module to define a - name: "my_password2" prompt: "Enter password2" private: yes - encrypt: "md5_crypt" + encrypt: "sha512_crypt" confirm: yes salt_size: 7 diff --git a/docsite/rst/playbooks_variables.rst b/docsite/rst/playbooks_variables.rst index 9c90a9afe2a..f9e3dda4e2a 100644 --- a/docsite/rst/playbooks_variables.rst +++ b/docsite/rst/playbooks_variables.rst @@ -327,9 +327,9 @@ To work with Base64 encoded strings:: {{ encoded | b64decode }} {{ decoded | b64encode }} -To take an md5sum of a filename:: +To take a sha1sum of a filename:: - {{ filename | md5 }} + {{ filename | sha1 }} To cast values as certain types, such as when you input a string as "True" from a vars_prompt and the system doesn't know it is a boolean value:: diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 8a4548dc169..b8cfea2014a 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -87,8 +87,13 @@ except ImportError: HAVE_HASHLIB=False try: - from hashlib import md5 as _md5 + from hashlib import sha1 as _sha1 HAVE_HASHLIB=True +except ImportError: + from sha import sha as _sha1 + +try: + from hashlib import md5 as _md5 except ImportError: from md5 import md5 as _md5 @@ -1236,6 +1241,10 @@ class AnsibleModule(object): ''' Return MD5 hex digest of local file using digest_from_file(). ''' return self.digest_from_file(filename, _md5()) + def sha1(self, filename): + ''' Return SHA1 hex digest of local file using digest_from_file(). ''' + return self.digest_from_file(filename, _sha1()) + def sha256(self, filename): ''' Return SHA-256 hex digest of local file using digest_from_file(). ''' if not HAVE_HASHLIB: diff --git a/lib/ansible/modules/core b/lib/ansible/modules/core index 2970b339eb8..6317d3a988f 160000 --- a/lib/ansible/modules/core +++ b/lib/ansible/modules/core @@ -1 +1 @@ -Subproject commit 2970b339eb8ea6031e6153cabe45459bc2bd5754 +Subproject commit 6317d3a988f7269340cb7a0d105d2c671ca1cd1e diff --git a/lib/ansible/modules/extras b/lib/ansible/modules/extras index ad181b7aa94..5a514ccddae 160000 --- a/lib/ansible/modules/extras +++ b/lib/ansible/modules/extras @@ -1 +1 @@ -Subproject commit ad181b7aa949848e3085065e09195cb28c34fdf7 +Subproject commit 5a514ccddae85ccc5802eea8751401600e45c32f diff --git a/lib/ansible/runner/__init__.py b/lib/ansible/runner/__init__.py index 4ef6f0ceab1..76412005441 100644 --- a/lib/ansible/runner/__init__.py +++ b/lib/ansible/runner/__init__.py @@ -53,9 +53,9 @@ from ansible.utils import update_hash module_replacer = ModuleReplacer(strip_comments=False) try: - from hashlib import md5 as _md5 + from hashlib import sha1 except ImportError: - from md5 import md5 as _md5 + from sha import sha as sha1 HAS_ATFORK=True try: @@ -209,7 +209,7 @@ class Runner(object): self.su_user_var = su_user self.su_user = None self.su_pass = su_pass - self.omit_token = '__omit_place_holder__%s' % _md5(os.urandom(64)).hexdigest() + self.omit_token = '__omit_place_holder__%s' % sha1(os.urandom(64)).hexdigest() self.vault_pass = vault_pass self.no_log = no_log self.run_once = run_once @@ -1159,26 +1159,29 @@ class Runner(object): # ***************************************************** - def _remote_md5(self, conn, tmp, path): - ''' takes a remote md5sum without requiring python, and returns 1 if no file ''' - cmd = conn.shell.md5(path) + def _remote_checksum(self, conn, tmp, path): + ''' takes a remote checksum and returns 1 if no file ''' + inject = self.get_inject_vars(conn.host) + hostvars = HostVars(inject['combined_cache'], self.inventory, vault_password=self.vault_pass) + python_interp = hostvars[conn.host].get('ansible_python_interpreter', 'python') + cmd = conn.shell.checksum(path, python_interp) data = self._low_level_exec_command(conn, cmd, tmp, sudoable=True) data2 = utils.last_non_blank_line(data['stdout']) try: if data2 == '': # this may happen if the connection to the remote server - # failed, so just return "INVALIDMD5SUM" to avoid errors - return "INVALIDMD5SUM" + # failed, so just return "INVALIDCHECKSUM" to avoid errors + return "INVALIDCHECKSUM" else: return data2.split()[0] except IndexError: - sys.stderr.write("warning: md5sum command failed unusually, please report this to the list so it can be fixed\n") - sys.stderr.write("command: %s\n" % md5s) + sys.stderr.write("warning: Calculating checksum failed unusually, please report this to the list so it can be fixed\n") + sys.stderr.write("command: %s\n" % cmd) sys.stderr.write("----\n") sys.stderr.write("output: %s\n" % data) sys.stderr.write("----\n") # this will signal that it changed and allow things to keep going - return "INVALIDMD5SUM" + return "INVALIDCHECKSUM" # ***************************************************** diff --git a/lib/ansible/runner/action_plugins/assemble.py b/lib/ansible/runner/action_plugins/assemble.py index c6f7165d822..9f5d450c2f8 100644 --- a/lib/ansible/runner/action_plugins/assemble.py +++ b/lib/ansible/runner/action_plugins/assemble.py @@ -108,10 +108,10 @@ class ActionModule(object): # Does all work assembling the file path = self._assemble_from_fragments(src, delimiter, _re) - pathmd5 = utils.md5s(path) - remote_md5 = self.runner._remote_md5(conn, tmp, dest) + path_checksum = utils.checksum_s(path) + remote_checksum = self.runner._remote_checksum(conn, tmp, dest) - if pathmd5 != remote_md5: + if path_checksum != remote_checksum: resultant = file(path).read() if self.runner.diff: dest_result = self.runner._execute_module(conn, tmp, 'slurp', "path=%s" % dest, inject=inject, persist_files=True) diff --git a/lib/ansible/runner/action_plugins/copy.py b/lib/ansible/runner/action_plugins/copy.py index 27b17b9969c..2b3d3871735 100644 --- a/lib/ansible/runner/action_plugins/copy.py +++ b/lib/ansible/runner/action_plugins/copy.py @@ -158,11 +158,11 @@ class ActionModule(object): tmp_path = self.runner._make_tmp_path(conn) for source_full, source_rel in source_files: - # Generate the MD5 hash of the local file. - local_md5 = utils.md5(source_full) + # Generate a hash of the local file. + local_checksum = utils.checksum(source_full) - # If local_md5 is not defined we can't find the file so we should fail out. - if local_md5 is None: + # If local_checksum is not defined we can't find the file so we should fail out. + if local_checksum is None: result = dict(failed=True, msg="could not find src=%s" % source_full) return ReturnData(conn=conn, result=result) @@ -174,27 +174,27 @@ class ActionModule(object): else: dest_file = conn.shell.join_path(dest) - # Attempt to get the remote MD5 Hash. - remote_md5 = self.runner._remote_md5(conn, tmp_path, dest_file) + # Attempt to get the remote checksum + remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file) - if remote_md5 == '3': - # The remote_md5 was executed on a directory. + if remote_checksum == '3': + # The remote_checksum was executed on a directory. if content is not None: # If source was defined as content remove the temporary file and fail out. self._remove_tempfile_if_content_defined(content, content_tempfile) result = dict(failed=True, msg="can not use content with a dir as dest") return ReturnData(conn=conn, result=result) else: - # Append the relative source location to the destination and retry remote_md5. + # Append the relative source location to the destination and retry remote_checksum dest_file = conn.shell.join_path(dest, source_rel) - remote_md5 = self.runner._remote_md5(conn, tmp_path, dest_file) + remote_checksum = self.runner._remote_checksum(conn, tmp_path, dest_file) - if remote_md5 != '1' and not force: + if remote_checksum != '1' and not force: # remote_file does not exist so continue to next iteration. continue - if local_md5 != remote_md5: - # The MD5 hashes don't match and we will change or error out. + if local_checksum != remote_checksum: + # The checksums don't match and we will change or error out. changed = True # Create a tmp_path if missing only if this is not recursive. @@ -254,7 +254,7 @@ class ActionModule(object): module_executed = True else: - # no need to transfer the file, already correct md5, but still need to call + # no need to transfer the file, already correct hash, but still need to call # the file module in case we want to change attributes self._remove_tempfile_if_content_defined(content, content_tempfile) @@ -283,8 +283,8 @@ class ActionModule(object): module_executed = True module_result = module_return.result - if not module_result.get('md5sum'): - module_result['md5sum'] = local_md5 + if not module_result.get('checksum'): + module_result['checksum'] = local_checksum if module_result.get('failed') == True: return module_return if module_result.get('changed') == True: diff --git a/lib/ansible/runner/action_plugins/fetch.py b/lib/ansible/runner/action_plugins/fetch.py index 80e8a89936f..825023a0bc9 100644 --- a/lib/ansible/runner/action_plugins/fetch.py +++ b/lib/ansible/runner/action_plugins/fetch.py @@ -50,26 +50,40 @@ class ActionModule(object): flat = utils.boolean(flat) fail_on_missing = options.get('fail_on_missing', False) fail_on_missing = utils.boolean(fail_on_missing) - validate_md5 = options.get('validate_md5', True) - validate_md5 = utils.boolean(validate_md5) + validate_checksum = options.get('validate_checksum', None) + if validate_checksum is not None: + validate_checksum = utils.boolean(validate_checksum) + # Alias for validate_checksum (old way of specifying it) + validate_md5 = options.get('validate_md5', None) + if validate_md5 is not None: + validate_md5 = utils.boolean(validate_md5) + if validate_md5 is None and validate_checksum is None: + # Default + validate_checksum = True + elif validate_checksum is None: + validate_checksum = validate_md5 + elif validate_md5 is not None and validate_checksum is not None: + results = dict(failed=True, msg="validate_checksum and validate_md5 cannot both be specified") + return ReturnData(conn, result=results) + if source is None or dest is None: results = dict(failed=True, msg="src and dest are required") return ReturnData(conn=conn, result=results) source = conn.shell.join_path(source) - # calculate md5 sum for the remote file - remote_md5 = self.runner._remote_md5(conn, tmp, source) + # calculate checksum for the remote file + remote_checksum = self.runner._remote_checksum(conn, tmp, source) # use slurp if sudo and permissions are lacking remote_data = None - if remote_md5 in ('1', '2') or self.runner.sudo: + if remote_checksum in ('1', '2') or self.runner.sudo: slurpres = self.runner._execute_module(conn, tmp, 'slurp', 'src=%s' % source, inject=inject) if slurpres.is_successful(): if slurpres.result['encoding'] == 'base64': remote_data = base64.b64decode(slurpres.result['content']) if remote_data is not None: - remote_md5 = utils.md5s(remote_data) + remote_checksum = utils.checksum_s(remote_data) # the source path may have been expanded on the # target system, so we compare it here and use the # expanded version if it's different @@ -101,23 +115,23 @@ class ActionModule(object): # these don't fail because you may want to transfer a log file that possibly MAY exist # but keep going to fetch other log files - if remote_md5 == '0': + if remote_checksum == '0': result = dict(msg="unable to calculate the md5 sum of the remote file", file=source, changed=False) return ReturnData(conn=conn, result=result) - if remote_md5 == '1': + if remote_checksum == '1': if fail_on_missing: result = dict(failed=True, msg="the remote file does not exist", file=source) else: result = dict(msg="the remote file does not exist, not transferring, ignored", file=source, changed=False) return ReturnData(conn=conn, result=result) - if remote_md5 == '2': + if remote_checksum == '2': result = dict(msg="no read permission on remote file, not transferring, ignored", file=source, changed=False) return ReturnData(conn=conn, result=result) - # calculate md5 sum for the local file - local_md5 = utils.md5(dest) + # calculate checksum for the local file + local_checksum = utils.checksum(dest) - if remote_md5 != local_md5: + if remote_checksum != local_checksum: # create the containing directories, if needed if not os.path.isdir(os.path.dirname(dest)): os.makedirs(os.path.dirname(dest)) @@ -129,13 +143,27 @@ class ActionModule(object): f = open(dest, 'w') f.write(remote_data) f.close() - new_md5 = utils.md5(dest) - if validate_md5 and new_md5 != remote_md5: - result = dict(failed=True, md5sum=new_md5, msg="md5 mismatch", file=source, dest=dest, remote_md5sum=remote_md5) + new_checksum = utils.secure_hash(dest) + # For backwards compatibility. We'll return None on FIPS enabled + # systems + try: + new_md5 = utils.md5(dest) + except ValueError: + new_md5 = None + + if validate_checksum and new_checksum != remote_checksum: + result = dict(failed=True, md5sum=new_md5, msg="checksum mismatch", file=source, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum) return ReturnData(conn=conn, result=result) - result = dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=remote_md5) + result = dict(changed=True, md5sum=new_md5, dest=dest, remote_md5sum=None, checksum=new_checksum, remote_checksum=remote_checksum) return ReturnData(conn=conn, result=result) else: - result = dict(changed=False, md5sum=local_md5, file=source, dest=dest) + # For backwards compatibility. We'll return None on FIPS enabled + # systems + try: + local_md5 = utils.md5(dest) + except ValueError: + local_md5 = None + + result = dict(changed=False, md5sum=local_md5, file=source, dest=dest, checksum=local_checksum) return ReturnData(conn=conn, result=result) diff --git a/lib/ansible/runner/action_plugins/template.py b/lib/ansible/runner/action_plugins/template.py index 4f5a41df8a0..2fe07c30394 100644 --- a/lib/ansible/runner/action_plugins/template.py +++ b/lib/ansible/runner/action_plugins/template.py @@ -87,10 +87,10 @@ class ActionModule(object): result = dict(failed=True, msg=type(e).__name__ + ": " + str(e)) return ReturnData(conn=conn, comm_ok=False, result=result) - local_md5 = utils.md5s(resultant) - remote_md5 = self.runner._remote_md5(conn, tmp, dest) + local_checksum = utils.checksum_s(resultant) + remote_checksum = self.runner._remote_checksum(conn, tmp, dest) - if local_md5 != remote_md5: + if local_checksum != remote_checksum: # template is different from the remote value diff --git a/lib/ansible/runner/action_plugins/unarchive.py b/lib/ansible/runner/action_plugins/unarchive.py index a569403cac3..1f831e42074 100644 --- a/lib/ansible/runner/action_plugins/unarchive.py +++ b/lib/ansible/runner/action_plugins/unarchive.py @@ -62,8 +62,8 @@ class ActionModule(object): else: source = utils.path_dwim(self.runner.basedir, source) - remote_md5 = self.runner._remote_md5(conn, tmp, dest) - if remote_md5 != '3': + remote_checksum = self.runner._remote_checksum(conn, tmp, dest) + if remote_checksum != '3': result = dict(failed=True, msg="dest '%s' must be an existing dir" % dest) return ReturnData(conn=conn, result=result) diff --git a/lib/ansible/runner/filter_plugins/core.py b/lib/ansible/runner/filter_plugins/core.py index 61b80bce2c5..e2a13f8c4e9 100644 --- a/lib/ansible/runner/filter_plugins/core.py +++ b/lib/ansible/runner/filter_plugins/core.py @@ -26,7 +26,7 @@ import re import collections import operator as py_operator from ansible import errors -from ansible.utils import md5s +from ansible.utils import md5s, checksum_s from distutils.version import LooseVersion, StrictVersion from random import SystemRandom from jinja2.filters import environmentfilter @@ -281,8 +281,13 @@ class FilterModule(object): # quote string for shell usage 'quote': quote, + # hash filters # md5 hex digest of string 'md5': md5s, + # sha1 hex digeset of string + 'sha1': checksum_s, + # checksum of string as used by ansible for checksuming files + 'checksum': checksum_s, # file glob 'fileglob': fileglob, diff --git a/lib/ansible/runner/shell_plugins/sh.py b/lib/ansible/runner/shell_plugins/sh.py index 1ee225830b5..134c857f171 100644 --- a/lib/ansible/runner/shell_plugins/sh.py +++ b/lib/ansible/runner/shell_plugins/sh.py @@ -59,23 +59,17 @@ class ShellModule(object): cmd += ' && echo %s' % basetmp return cmd - def md5(self, path): + def checksum(self, path, python_interp): path = pipes.quote(path) # The following test needs to be SH-compliant. BASH-isms will # not work if /bin/sh points to a non-BASH shell. test = "rc=0; [ -r \"%s\" ] || rc=2; [ -f \"%s\" ] || rc=1; [ -d \"%s\" ] && echo 3 && exit 0" % ((path,) * 3) - md5s = [ - "(/usr/bin/md5sum %s 2>/dev/null)" % path, # Linux - "(/sbin/md5sum -q %s 2>/dev/null)" % path, # ? - "(/usr/bin/digest -a md5 %s 2>/dev/null)" % path, # Solaris 10+ - "(/sbin/md5 -q %s 2>/dev/null)" % path, # Freebsd - "(/usr/bin/md5 -n %s 2>/dev/null)" % path, # Netbsd - "(/bin/md5 -q %s 2>/dev/null)" % path, # Openbsd - "(/usr/bin/csum -h MD5 %s 2>/dev/null)" % path, # AIX - "(/bin/csum -h MD5 %s 2>/dev/null)" % path # AIX also + csums = [ + "(%s -c 'import hashlib; print(hashlib.sha1(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path), # Python > 2.4 (including python3) + "(%s -c 'import sha; print(sha.sha(open(\"%s\", \"rb\").read()).hexdigest())' 2>/dev/null)" % (python_interp, path), # Python == 2.4 ] - cmd = " || ".join(md5s) + cmd = " || ".join(csums) cmd = "%s; %s || (echo \"${rc} %s\")" % (test, cmd, path) return cmd diff --git a/lib/ansible/utils/__init__.py b/lib/ansible/utils/__init__.py index 952e8537d0b..e82ae8d3749 100644 --- a/lib/ansible/utils/__init__.py +++ b/lib/ansible/utils/__init__.py @@ -68,6 +68,14 @@ try: except ImportError: import simplejson as json +# Note, sha1 is the only hash algorithm compatible with python2.4 and with +# FIPS-140 mode (as of 11-2014) +try: + from hashlib import sha1 as sha1 +except ImportError: + from sha import sha as sha1 + +# Backwards compat only try: from hashlib import md5 as _md5 except ImportError: @@ -821,22 +829,22 @@ def merge_hash(a, b): return result -def md5s(data): - ''' Return MD5 hex digest of data. ''' +def secure_hash_s(data, hash_func=sha1): + ''' Return a secure hash hex digest of data. ''' - digest = _md5() + digest = hash_func() try: digest.update(data) except UnicodeEncodeError: digest.update(data.encode('utf-8')) return digest.hexdigest() -def md5(filename): - ''' Return MD5 hex digest of local file, None if file is not present or a directory. ''' +def secure_hash(filename, hash_func=sha1): + ''' Return a secure hash hex digest of local file, None if file is not present or a directory. ''' if not os.path.exists(filename) or os.path.isdir(filename): return None - digest = _md5() + digest = hash_func() blocksize = 64 * 1024 try: infile = open(filename, 'rb') @@ -849,6 +857,19 @@ def md5(filename): raise errors.AnsibleError("error while accessing the file %s, error was: %s" % (filename, e)) return digest.hexdigest() +# The checksum algorithm must match with the algorithm in ShellModule.checksum() method +checksum = secure_hash +checksum_s = secure_hash_s + +# Backwards compat. Some modules include md5s in their return values +# Continue to support that for now. As of ansible-1.8, all of those modules +# should also return "checksum" (sha1 for now) +def md5s(data): + return secure_hash_s(data, _md5) + +def md5(filename): + return secure_hash(filename, _md5) + def default(value, function): ''' syntactic sugar around lazy evaluation of defaults ''' if value is None: diff --git a/lib/ansible/utils/vault.py b/lib/ansible/utils/vault.py index 50b686c1e04..ad2dfab0b76 100644 --- a/lib/ansible/utils/vault.py +++ b/lib/ansible/utils/vault.py @@ -26,6 +26,8 @@ from io import BytesIO from subprocess import call from ansible import errors from hashlib import sha256 +# Note: Only used for loading obsolete VaultAES files. All files are written +# using the newer VaultAES256 which does not require md5 from hashlib import md5 from binascii import hexlify from binascii import unhexlify diff --git a/test/integration/roles/test_assemble/tasks/main.yml b/test/integration/roles/test_assemble/tasks/main.yml index f06cee6ace8..d0c1f15e56d 100644 --- a/test/integration/roles/test_assemble/tasks/main.yml +++ b/test/integration/roles/test_assemble/tasks/main.yml @@ -37,7 +37,19 @@ assert: that: - "result.state == 'file'" - - "result.md5sum == '96905702a2ece40de6bf3a94b5062513'" + - "result.changed == True" + - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'" + +- name: test assemble with all fragments + assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled1" + register: result + +- name: assert that the same assemble made no changes + assert: + that: + - "result.state == 'file'" + - "result.changed == False" + - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'" - name: test assemble with fragments matching a regex assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled2" regexp="^fragment[1-3]$" @@ -47,7 +59,7 @@ assert: that: - "result.state == 'file'" - - "result.md5sum == 'eb9e3486a9cd6943b5242e573b9b9349'" + - "result.checksum == 'edfe2d7487ef8f5ebc0f1c4dc57ba7b70a7b8e2b'" - name: test assemble with a delimiter assemble: src="{{output_dir}}/src" dest="{{output_dir}}/assembled3" delimiter="#--- delimiter ---#" @@ -57,7 +69,7 @@ assert: that: - "result.state == 'file'" - - "result.md5sum == '4773eac67aba3f0be745876331c8a450'" + - "result.checksum == '505359f48c65b3904127cf62b912991d4da7ed6d'" - name: test assemble with remote_src=False assemble: src="./" dest="{{output_dir}}/assembled4" remote_src=no @@ -67,7 +79,7 @@ assert: that: - "result.state == 'file'" - - "result.md5sum == '96905702a2ece40de6bf3a94b5062513'" + - "result.checksum == '048a1bd1951aa5ccc427eeb4ca19aee45e9c68b3'" - name: test assemble with remote_src=False and a delimiter assemble: src="./" dest="{{output_dir}}/assembled5" remote_src=no delimiter="#--- delimiter ---#" @@ -77,5 +89,5 @@ assert: that: - "result.state == 'file'" - - "result.md5sum == '4773eac67aba3f0be745876331c8a450'" + - "result.checksum == '505359f48c65b3904127cf62b912991d4da7ed6d'" diff --git a/test/integration/roles/test_command_shell/tasks/main.yml b/test/integration/roles/test_command_shell/tasks/main.yml index 3c273260c11..b331452b7c6 100644 --- a/test/integration/roles/test_command_shell/tasks/main.yml +++ b/test/integration/roles/test_command_shell/tasks/main.yml @@ -185,7 +185,7 @@ "multiline echo" \ "with a new line in quotes" \ - | md5sum \ + | sha1sum \ | tr -s ' ' \ | cut -f1 -d ' ' echo "this is a second line" @@ -197,7 +197,7 @@ assert: that: - "shell_result6.changed" - - "shell_result6.stdout == '32f3cc201b69ed8afa3902b80f554ca8\nthis is a second line'" + - "shell_result6.stdout == '5575bb6b71c9558db0b6fbbf2f19909eeb4e3b98\nthis is a second line'" - name: execute a shell command using a literal multiline block with arguments in it shell: | diff --git a/test/integration/roles/test_copy/tasks/main.yml b/test/integration/roles/test_copy/tasks/main.yml index 47ed5166578..fa09d37eb44 100644 --- a/test/integration/roles/test_copy/tasks/main.yml +++ b/test/integration/roles/test_copy/tasks/main.yml @@ -40,6 +40,7 @@ - "'group' in copy_result" - "'gid' in copy_result" - "'md5sum' in copy_result" + - "'checksum' in copy_result" - "'owner' in copy_result" - "'size' in copy_result" - "'src' in copy_result" @@ -51,10 +52,11 @@ that: - "copy_result.changed == true" -- name: verify that the file md5sum is correct - assert: - that: +- name: verify that the file checksums are correct + assert: + that: - "copy_result.md5sum == 'c47397529fe81ab62ba3f85e9f4c71f2'" + - "copy_result.checksum == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'" - name: check the stat results of the file stat: path={{output_file}} @@ -71,6 +73,7 @@ - "stat_results.stat.isreg == true" - "stat_results.stat.issock == false" - "stat_results.stat.md5 == 'c47397529fe81ab62ba3f85e9f4c71f2'" + - "stat_results.stat.checksum == 'c79a6506c1c948be0d456ab5104d5e753ab2f3e6'" - name: overwrite the file via same means copy: src=foo.txt dest={{output_file}} @@ -180,7 +183,7 @@ that: - "copy_result6.changed" - "copy_result6.dest == '{{output_dir|expanduser}}/multiline.txt'" - - "copy_result6.md5sum == '1627d51e7e607c92cf1a502bf0c6cce3'" + - "copy_result6.checksum == '9cd0697c6a9ff6689f0afb9136fa62e0b3fee903'" # test overwriting a file as an unprivileged user (pull request #8624) # this can't be relative to {{output_dir}} as ~root usually has mode 700 @@ -202,7 +205,7 @@ that: - "copy_result7.changed" - "copy_result7.dest == '/tmp/worldwritable/file.txt'" - - "copy_result7.md5sum == '73feffa4b7f6bb68e44cf984c85f6e88'" + - "copy_result7.checksum == 'bbe960a25ea311d21d40669e93df2003ba9b90a2'" - name: clean up file: dest=/tmp/worldwritable state=absent @@ -230,10 +233,10 @@ - stat_link_result.stat.islnk - name: get the md5 of the link target - shell: md5sum {{output_dir}}/follow_test | cut -f1 -sd ' ' + shell: sha1sum {{output_dir}}/follow_test | cut -f1 -sd ' ' register: target_file_result - name: assert that the link target was updated assert: that: - - replace_follow_result.md5sum == target_file_result.stdout + - replace_follow_result.checksum == target_file_result.stdout diff --git a/test/integration/roles/test_lineinfile/tasks/main.yml b/test/integration/roles/test_lineinfile/tasks/main.yml index 8d58cbba6f2..3f8a8dc5bad 100644 --- a/test/integration/roles/test_lineinfile/tasks/main.yml +++ b/test/integration/roles/test_lineinfile/tasks/main.yml @@ -24,7 +24,7 @@ assert: that: - "result.changed == true" - - "result.md5sum == '6be7fb7fa7fb758c80a6dc0722979c40'" + - "result.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'" - "result.state == 'file'" - name: insert a line at the beginning of the file, and back it up @@ -42,19 +42,19 @@ stat: path={{result.backup}} register: result -- name: assert the backup file matches the previous md5 +- name: assert the backup file matches the previous hash assert: that: - - "result.stat.md5 == '6be7fb7fa7fb758c80a6dc0722979c40'" + - "result.stat.checksum == '5feac65e442c91f557fc90069ce6efc4d346ab51'" - name: stat the test after the insert at the head stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the insert at the head +- name: assert test hash is what we expect for the file with the insert at the head assert: that: - - "result.stat.md5 == '07c16434644a2a3cc1807c685917443a'" + - "result.stat.checksum == '7eade4042b23b800958fe807b5bfc29f8541ec09'" - name: insert a line at the end of the file lineinfile: dest={{output_dir}}/test.txt state=present line="New line at the end" insertafter="EOF" @@ -70,10 +70,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the insert at the end +- name: assert test checksum matches after the insert at the end assert: that: - - "result.stat.md5 == 'da4c2150e5782fcede1840280ab87eff'" + - "result.stat.checksum == 'fb57af7dc10a1006061b000f1f04c38e4bef50a9'" - name: insert a line after the first line lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 1" insertafter="^This is line 1$" @@ -89,10 +89,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the insert after the first line +- name: assert test checksum matches after the insert after the first line assert: that: - - "result.stat.md5 == '196722c8faaa28b960bee66fa4cce58c'" + - "result.stat.checksum == '5348da605b1bc93dbadf3a16474cdf22ef975bec'" - name: insert a line before the last line lineinfile: dest={{output_dir}}/test.txt state=present line="New line after line 5" insertbefore="^This is line 5$" @@ -108,10 +108,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the insert before the last line +- name: assert test checksum matches after the insert before the last line assert: that: - - "result.stat.md5 == 'd5955ee042139dfef16dbe3a7334475f'" + - "result.stat.checksum == 'e1cae425403507feea4b55bb30a74decfdd4a23e'" - name: replace a line with backrefs lineinfile: dest={{output_dir}}/test.txt state=present line="This is line 3" backrefs=yes regexp="^(REF) .* \\1$" @@ -127,16 +127,16 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after backref line was replaced +- name: assert test checksum matches after backref line was replaced assert: that: - - "result.stat.md5 == '0f585270054e17be242743dd31c6f593'" + - "result.stat.checksum == '2ccdf45d20298f9eaece73b713648e5489a52444'" - name: remove the middle line lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 3$" register: result -- name: assert that the line was inserted at the head of the file +- name: assert that the line was removed assert: that: - "result.changed == true" @@ -146,10 +146,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the middle line was removed +- name: assert test checksum matches after the middle line was removed assert: that: - - "result.stat.md5 == '661603660051991b79429c2dc68d9a67'" + - "result.stat.checksum == 'a6ba6865547c19d4c203c38a35e728d6d1942c75'" - name: run a validation script that succeeds lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 5$" validate="true %s" @@ -165,10 +165,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after the validation succeeded +- name: assert test checksum matches after the validation succeeded assert: that: - - "result.stat.md5 == '9af984939bd859f7794661e501b4f1a4'" + - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'" - name: run a validation script that fails lineinfile: dest={{output_dir}}/test.txt state=absent regexp="^This is line 1$" validate="/bin/false %s" @@ -184,10 +184,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches the previous after the validation failed +- name: assert test checksum matches the previous after the validation failed assert: that: - - "result.stat.md5 == '9af984939bd859f7794661e501b4f1a4'" + - "result.stat.checksum == '76955a4516a00a38aad8427afc9ee3e361024ba5'" - name: use create=yes lineinfile: dest={{output_dir}}/new_test.txt create=yes insertbefore=BOF state=present line="This is a new file" @@ -204,10 +204,10 @@ register: result ignore_errors: yes -- name: assert the newly created test md5 matches +- name: assert the newly created test checksum matches assert: that: - - "result.stat.md5 == 'fef1d487711facfd7aa2c87d788c19d9'" + - "result.stat.checksum == '038f10f9e31202451b093163e81e06fbac0c6f3a'" # Test EOF in cases where file has no newline at EOF - name: testnoeof deploy the file for lineinfile @@ -238,10 +238,10 @@ stat: path={{output_dir}}/testnoeof.txt register: result -- name: testnoeof assert test md5 matches after the insert at the end +- name: testnoeof assert test checksum matches after the insert at the end assert: that: - - "result.stat.md5 == 'f75c9d51f45afd7295000e63ce655220'" + - "result.stat.checksum == 'f9af7008e3cb67575ce653d094c79cabebf6e523'" # Test EOF with empty file to make sure no unneccessary newline is added - name: testempty deploy the testempty file for lineinfile @@ -262,18 +262,18 @@ stat: path={{output_dir}}/testempty.txt register: result -- name: testempty assert test md5 matches after the insert at the end +- name: testempty assert test checksum matches after the insert at the end assert: that: - - "result.stat.md5 == '357dcbee8dfb4436f63bab00a235c45a'" + - "result.stat.checksum == 'f440dc65ea9cec3fd496c1479ddf937e1b949412'" - stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after insert the multiple lines +- name: assert test checksum matches after inserting multiple lines assert: that: - - "result.stat.md5 == 'c2510d5bc8fdef8e752b8f8e74c784c2'" + - "result.stat.checksum == 'bf5b711f8f0509355aaeb9d0d61e3e82337c1365'" - name: replace a line with backrefs included in the line lineinfile: dest={{output_dir}}/test.txt state=present line="New \\1 created with the backref" backrefs=yes regexp="^This is (line 4)$" @@ -289,10 +289,10 @@ stat: path={{output_dir}}/test.txt register: result -- name: assert test md5 matches after backref line was replaced +- name: assert test checksum matches after backref line was replaced assert: that: - - "result.stat.md5 == '65f955c2a9722fd43d07103d7756ff9b'" + - "result.stat.checksum == '04b7a54d0fb233a4e26c9e625325bb4874841b3c'" ################################################################### # issue 8535 @@ -332,10 +332,10 @@ stat: path={{output_dir}}/test_quoting.txt register: result -- name: assert test md5 matches after backref line was replaced +- name: assert test checksum matches after backref line was replaced assert: that: - - "result.stat.md5 == '29f349baf1b9c6703beeb346fe8dc669'" + - "result.stat.checksum == '7dc3cb033c3971e73af0eaed6623d4e71e5743f1'" - name: insert a line into the quoted file with a single quote lineinfile: dest={{output_dir}}/test_quoting.txt line="import g'" @@ -350,9 +350,9 @@ stat: path={{output_dir}}/test_quoting.txt register: result -- name: assert test md5 matches after backref line was replaced +- name: assert test checksum matches after backref line was replaced assert: that: - - "result.stat.md5 == 'fbe9c4ba2490f70eb1974ce31ec4a39f'" + - "result.stat.checksum == '73b271c2cc1cef5663713bc0f00444b4bf9f4543'" ################################################################### diff --git a/test/integration/roles/test_service/tasks/main.yml b/test/integration/roles/test_service/tasks/main.yml index ab4335a8a52..6f941eeb5c1 100644 --- a/test/integration/roles/test_service/tasks/main.yml +++ b/test/integration/roles/test_service/tasks/main.yml @@ -6,7 +6,7 @@ assert: that: - "install_result.dest == '/usr/sbin/ansible_test_service'" - - "install_result.md5sum == '9ad49eaf390b30b1206b793ec71200ed'" + - "install_result.checksum == 'baaa79448a976922c080f1971321d203c6df0961'" - "install_result.state == 'file'" - "install_result.mode == '0755'" diff --git a/test/integration/roles/test_service/tasks/systemd_setup.yml b/test/integration/roles/test_service/tasks/systemd_setup.yml index 6d429332131..4a3a81a4a60 100644 --- a/test/integration/roles/test_service/tasks/systemd_setup.yml +++ b/test/integration/roles/test_service/tasks/systemd_setup.yml @@ -12,7 +12,7 @@ - "install_systemd_result.dest == '/usr/lib/systemd/system/ansible_test.service'" - "install_systemd_result.state == 'file'" - "install_systemd_result.mode == '0644'" - - "install_systemd_result.md5sum == '6be64a1e44e9e72a467e70a0b562444f'" + - "install_systemd_result.checksum == 'ca4b413fdf3cb2002f51893b9e42d2e449ec5afb'" - "install_broken_systemd_result.dest == '/usr/lib/systemd/system/ansible_test_broken.service'" - "install_broken_systemd_result.state == 'link'" diff --git a/test/integration/roles/test_service/tasks/sysv_setup.yml b/test/integration/roles/test_service/tasks/sysv_setup.yml index 83a1d6a8c48..1bc9dbc3711 100644 --- a/test/integration/roles/test_service/tasks/sysv_setup.yml +++ b/test/integration/roles/test_service/tasks/sysv_setup.yml @@ -8,5 +8,5 @@ - "install_sysv_result.dest == '/etc/init.d/ansible_test'" - "install_sysv_result.state == 'file'" - "install_sysv_result.mode == '0755'" - - "install_sysv_result.md5sum == 'ebf6a9064ca8628187f3a6caf8e2a279'" + - "install_sysv_result.md5sum == '174fa255735064b420600e4c8637ea0eff28d0c1'" diff --git a/test/integration/roles/test_service/tasks/upstart_setup.yml b/test/integration/roles/test_service/tasks/upstart_setup.yml index 118d2da50e1..e9607bb030e 100644 --- a/test/integration/roles/test_service/tasks/upstart_setup.yml +++ b/test/integration/roles/test_service/tasks/upstart_setup.yml @@ -12,8 +12,8 @@ - "install_upstart_result.dest == '/etc/init/ansible_test.conf'" - "install_upstart_result.state == 'file'" - "install_upstart_result.mode == '0644'" - - "install_upstart_result.md5sum == 'ab3900ea4de8423add764c12aeb90c01'" + - "install_upstart_result.checksum == '5c314837b6c4dd6c68d1809653a2974e9078e02a'" - "install_upstart_broken_result.dest == '/etc/init/ansible_broken_test.conf'" - "install_upstart_broken_result.state == 'file'" - "install_upstart_broken_result.mode == '0644'" - - "install_upstart_broken_result.md5sum == '015e183d10c311276c3e269cbeb309b7'" + - "install_upstart_broken_result.checksum == 'e66497894f2b2bf71e1380a196cc26089cc24a10'" diff --git a/test/integration/roles/test_stat/tasks/main.yml b/test/integration/roles/test_stat/tasks/main.yml index f27721a6979..b0b16d7f9eb 100644 --- a/test/integration/roles/test_stat/tasks/main.yml +++ b/test/integration/roles/test_stat/tasks/main.yml @@ -46,6 +46,8 @@ - "'isuid' in stat_result.stat" - "'md5' in stat_result.stat" - "stat_result.stat.md5 == '5eb63bbbe01eeed093cb22bb8f5acdc3'" + - "'checksum' in stat_result.stat" + - "stat_result.stat.checksum == '2aae6c35c94fcfb415dbe95f408b9ce91ee846ed'" - "'mode' in stat_result.stat" # why is this 420? - "'mtime' in stat_result.stat" - "'nlink' in stat_result.stat" diff --git a/test/integration/roles/test_template/tasks/main.yml b/test/integration/roles/test_template/tasks/main.yml index 03058854733..d7d812f3bab 100644 --- a/test/integration/roles/test_template/tasks/main.yml +++ b/test/integration/roles/test_template/tasks/main.yml @@ -27,6 +27,7 @@ - "'group' in template_result" - "'gid' in template_result" - "'md5sum' in template_result" + - "'checksum' in template_result" - "'owner' in template_result" - "'size' in template_result" - "'src' in template_result" diff --git a/test/units/TestModuleUtilsBasic.py b/test/units/TestModuleUtilsBasic.py index ceba17be4fd..f5962a94787 100644 --- a/test/units/TestModuleUtilsBasic.py +++ b/test/units/TestModuleUtilsBasic.py @@ -7,7 +7,7 @@ from nose.tools import timed from ansible import errors from ansible.module_common import ModuleReplacer -from ansible.utils import md5 as utils_md5 +from ansible.utils import checksum as utils_checksum TEST_MODULE_DATA = """ from ansible.module_utils.basic import * @@ -113,8 +113,8 @@ class TestModuleUtilsBasic(unittest.TestCase): (rc, out, err) = self.module.run_command('echo "foo bar" > %s' % tmp_path, use_unsafe_shell=True) self.assertEqual(rc, 0) self.assertTrue(os.path.exists(tmp_path)) - md5sum = utils_md5(tmp_path) - self.assertEqual(md5sum, '5ceaa7ed396ccb8e959c02753cb4bd18') + checksum = utils_checksum(tmp_path) + self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec') except: raise finally: @@ -127,8 +127,8 @@ class TestModuleUtilsBasic(unittest.TestCase): (rc, out, err) = self.module.run_command('echo "foo bar" >> %s' % tmp_path, use_unsafe_shell=True) self.assertEqual(rc, 0) self.assertTrue(os.path.exists(tmp_path)) - md5sum = utils_md5(tmp_path) - self.assertEqual(md5sum, '5ceaa7ed396ccb8e959c02753cb4bd18') + checksum = utils_checksum(tmp_path) + self.assertEqual(checksum, 'd53a205a336e07cf9eac45471b3870f9489288ec') except: raise finally: diff --git a/test/units/TestUtils.py b/test/units/TestUtils.py index af10a1e0553..178eaae50c9 100644 --- a/test/units/TestUtils.py +++ b/test/units/TestUtils.py @@ -366,6 +366,16 @@ class TestUtils(unittest.TestCase): self.assertEqual(ansible.utils.md5(os.path.join(os.path.dirname(__file__), 'ansible.cf')), None) + def test_checksum_s(self): + self.assertEqual(ansible.utils.checksum_s('ansible'), 'bef45157a43c9e5f469d188810814a4a8ab9f2ed') + # Need a test that causes UnicodeEncodeError See 4221 + + def test_checksum(self): + self.assertEqual(ansible.utils.checksum(os.path.join(os.path.dirname(__file__), 'ansible.cfg')), + '658b67c8ac7595adde7048425ff1f9aba270721a') + self.assertEqual(ansible.utils.md5(os.path.join(os.path.dirname(__file__), 'ansible.cf')), + None) + def test_default(self): self.assertEqual(ansible.utils.default(None, lambda: {}), {}) self.assertEqual(ansible.utils.default(dict(foo='bar'), lambda: {}), dict(foo='bar')) diff --git a/v2/ansible/parsing/vault/__init__.py b/v2/ansible/parsing/vault/__init__.py index 44f50f7d21e..92c99fdad5e 100644 --- a/v2/ansible/parsing/vault/__init__.py +++ b/v2/ansible/parsing/vault/__init__.py @@ -30,6 +30,8 @@ from io import BytesIO from subprocess import call from ansible import errors from hashlib import sha256 +# Note: Only used for loading obsolete VaultAES files. All files are written +# using the newer VaultAES256 which does not require md5 from hashlib import md5 from binascii import hexlify from binascii import unhexlify diff --git a/v2/ansible/playbook/role/__init__.py b/v2/ansible/playbook/role/__init__.py index 8f37970d59e..67485f0f9c2 100644 --- a/v2/ansible/playbook/role/__init__.py +++ b/v2/ansible/playbook/role/__init__.py @@ -23,7 +23,7 @@ from six import iteritems, string_types import os -from hashlib import md5 +from hashlib import sha1 from types import NoneType from ansible.errors import AnsibleError, AnsibleParserError @@ -39,7 +39,7 @@ __all__ = ['Role', 'ROLE_CACHE'] # The role cache is used to prevent re-loading roles, which -# may already exist. Keys into this cache are the MD5 hash +# may already exist. Keys into this cache are the SHA1 hash # of the role definition (for dictionary definitions, this # will be based on the repr() of the dictionary object) ROLE_CACHE = dict() @@ -60,7 +60,7 @@ class Role: self._handler_blocks = [] self._default_vars = dict() self._role_vars = dict() - + def __repr__(self): return self.get_name()