From 375a1eaf43da16bd86db375eab4c1deb0a8a1c1d Mon Sep 17 00:00:00 2001
From: John Kleint <jkleint@gmail.com>
Date: Mon, 9 Jul 2012 13:27:47 -0400
Subject: [PATCH] Use iterative MD5 hashing.

---
 lib/ansible/utils.py | 26 +++++++++++++++-----------
 library/assemble     | 29 +++++++++++++++--------------
 library/copy         | 27 +++++++++++++++------------
 library/setup        | 27 +++++++++++++++------------
 4 files changed, 60 insertions(+), 49 deletions(-)

diff --git a/lib/ansible/utils.py b/lib/ansible/utils.py
index 8ca0b718cf0..50765a430b4 100644
--- a/lib/ansible/utils.py
+++ b/lib/ansible/utils.py
@@ -33,11 +33,9 @@ except ImportError:
     import simplejson as json
 
 try:
-    import hashlib 
-    HAVE_HASHLIB=True
+    from hashlib import md5 as _md5
 except ImportError: 
-    import md5
-    HAVE_HASHLIB=False
+    from md5 import md5 as _md5
 
 from ansible import errors
 import ansible.constants as C
@@ -321,13 +319,19 @@ def parse_kv(args):
     return options
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
-     if not os.path.exists(filename):
-         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
+    if not os.path.exists(filename):
+        return None
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
+
 
 
 ####################################################################
diff --git a/library/assemble b/library/assemble
index 7c0f7c773a3..884e4a51b34 100755
--- a/library/assemble
+++ b/library/assemble
@@ -30,14 +30,10 @@ import syslog
 import tempfile
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
-except ImportError:
-    import md5
-    HAVE_HASHLIB=False
+    from hashlib import md5 as _md5
+except ImportError: 
+    from md5 import md5 as _md5
 
-# Since hashlib is only available in 2.5 and onwards, this module
-# uses md5 which is available in 2.4.
 
 # ===========================================
 # Support methods
@@ -66,13 +62,18 @@ def write_temp_file(data):
     return path
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
-     if not os.path.exists(filename):
-         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
+    if not os.path.exists(filename):
+        return None
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================
 
diff --git a/library/copy b/library/copy
index d0aaac09262..d2a6f750517 100755
--- a/library/copy
+++ b/library/copy
@@ -25,11 +25,9 @@ import shutil
 import syslog
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
-except ImportError:
-    import md5
-    HAVE_HASHLIB=False
+    from hashlib import md5 as _md5
+except ImportError: 
+    from md5 import md5 as _md5
 
 # ===========================================
 # convert arguments of form a=b c=d
@@ -46,13 +44,18 @@ def exit_kv(rc=0, **kwargs):
     sys.exit(rc)
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
-     if not os.path.exists(filename):
-         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
+    if not os.path.exists(filename):
+        return None
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================
 
diff --git a/library/setup b/library/setup
index bed214c70c6..4cc343b7b9c 100755
--- a/library/setup
+++ b/library/setup
@@ -34,11 +34,9 @@ import traceback
 import syslog
 
 try:
-    import hashlib
-    HAVE_HASHLIB=True
-except ImportError:
-    import md5
-    HAVE_HASHLIB=False
+    from hashlib import md5 as _md5
+except ImportError: 
+    from md5 import md5 as _md5
 
 try:
     import selinux
@@ -319,13 +317,18 @@ def ansible_facts():
     return facts
 
 def md5(filename):
-     ''' compute md5sum, return None if file is not present '''
-     if not os.path.exists(filename):
-         return None
-     if HAVE_HASHLIB:
-         return hashlib.md5(file(filename).read()).hexdigest()
-     else:
-         return md5.new(file(filename).read()).hexdigest()
+    ''' Return MD5 hex digest of local file, or None if file is not present. '''
+    if not os.path.exists(filename):
+        return None
+    digest = _md5()
+    blocksize = 64 * 1024
+    infile = open(filename, 'rb')
+    block = infile.read(blocksize)
+    while block:
+        digest.update(block)
+        block = infile.read(blocksize)
+    infile.close()
+    return digest.hexdigest()
 
 # ===========================================