From 84a59e472b9fd4c9d8bcd9ee47af231c113993ab Mon Sep 17 00:00:00 2001
From: Toshio Kuratomi <a.badger@gmail.com>
Date: Fri, 12 May 2017 18:04:48 -0700
Subject: [PATCH] Be explicit about pickle protocol and encoding (#24454)

On Python3 and Python2 use pickle slightly differently so we need to be
explicit about some things.

If pickles could be shared between python2 and python3, as in
ansible-connection and the pickle cache, we need to specify the protocol
to use when dumping and the encoding to use for byte strings when
loading.

The dumping protocol needs to be no higher than 2 as python-2 only
supports up to protocol 2.  The encoding should usually be 'bytes' so
that python2 str type becomes python3 bytes type.  However, doing this
means that we must make sure that the objects being serialized properly
make their strings into text strings except when they're supposed to be
bytes.  If strings are improperly byte strings, they may cause
tracebacks on the receiving end
---
 bin/ansible-connection              | 14 +++++++++-----
 lib/ansible/module_utils/service.py |  9 ++++++---
 lib/ansible/plugins/cache/pickle.py |  6 +++++-
 3 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/bin/ansible-connection b/bin/ansible-connection
index 676cd70223b..dcbcea0d949 100755
--- a/bin/ansible-connection
+++ b/bin/ansible-connection
@@ -41,8 +41,6 @@ import syslog
 import datetime
 import logging
 
-from io import BytesIO
-
 from ansible import constants as C
 from ansible.module_utils._text import to_bytes, to_native
 from ansible.module_utils.six import PY3
@@ -214,7 +212,10 @@ class Server():
                             display.display("socket operation is CONTEXT", log_only=True)
                             pc_data = data.split(b'CONTEXT: ', 1)[1]
 
-                            pc_data = cPickle.loads(pc_data)
+                            if PY3:
+                                pc_data = cPickle.loads(pc_data, encoding='bytes')
+                            else:
+                                pc_data = cPickle.loads(pc_data)
 
                             pc = PlayContext()
                             pc.deserialize(pc_data)
@@ -267,11 +268,14 @@ def main():
         cur_line = stdin.readline()
         init_data = b''
         while cur_line.strip() != b'#END_INIT#':
-            if cur_line  == b'':
+            if cur_line == b'':
                 raise Exception("EOF found before init data was complete")
             init_data += cur_line
             cur_line = stdin.readline()
-        pc_data = cPickle.loads(init_data)
+        if PY3:
+            pc_data = cPickle.loads(init_data, encoding='bytes')
+        else:
+            pc_data = cPickle.loads(init_data)
 
         pc = PlayContext()
         pc.deserialize(pc_data)
diff --git a/lib/ansible/module_utils/service.py b/lib/ansible/module_utils/service.py
index c8269f50ca4..dcf58e485b1 100644
--- a/lib/ansible/module_utils/service.py
+++ b/lib/ansible/module_utils/service.py
@@ -176,8 +176,8 @@ def daemonize(module, cmd):
         # even after fds close, we might want to wait for pid to die
         p.wait()
 
-        # Return a pickled data o parent
-        return_data = pickle.dumps([p.returncode, to_text(output[p.stdout]), to_text(output[p.stderr])])
+        # Return a pickled data of parent
+        return_data = pickle.dumps([p.returncode, to_text(output[p.stdout]), to_text(output[p.stderr])], protocol=pickle.HIGHEST_PROTOCOL)
         os.write(pipe[1], to_bytes(return_data, errors=errors))
 
         # clean up
@@ -202,7 +202,10 @@ def daemonize(module, cmd):
                     break
                 return_data += b(data)
 
-        return pickle.loads(to_text(return_data, errors=errors))
+        # Note: no need to specify encoding on py3 as this module sends the
+        # pickle to itself (thus same python interpreter so we aren't mixing
+        # py2 and py3)
+        return pickle.loads(to_bytes(return_data, errors=errors))
 
 def check_ps(module, pattern):
 
diff --git a/lib/ansible/plugins/cache/pickle.py b/lib/ansible/plugins/cache/pickle.py
index bf61071b0fb..3ac025170d2 100644
--- a/lib/ansible/plugins/cache/pickle.py
+++ b/lib/ansible/plugins/cache/pickle.py
@@ -33,6 +33,7 @@ try:
 except ImportError:
     import pickle
 
+from ansible.module_utils.six import PY3
 from ansible.plugins.cache import BaseFileCacheModule
 
 class CacheModule(BaseFileCacheModule):
@@ -43,7 +44,10 @@ class CacheModule(BaseFileCacheModule):
     def _load(self, filepath):
         # Pickle is a binary format
         with open(filepath, 'rb') as f:
-            return pickle.load(f)
+            if PY3:
+                return pickle.load(f, encoding='bytes')
+            else:
+                return pickle.load(f)
 
     def _dump(self, value, filepath):
         with open(filepath, 'wb') as f: