Handle the fact that in sqlite binary data might be stored as unicode or bytes

synapse/storage/_base.py

@@ -812,6 +812,9 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted, rejected_reason = res
 
+        internal_metadata = self.database_engine.load_unicode(internal_metadata)
+        js = self.database_engine.load_unicode(js)
+
         start_time = update_counter("select_event", start_time)
 
         result = self._get_event_from_row_txn(
@@ -839,11 +842,11 @@ class SQLBaseStore(object):
             return curr_time
 
         logger.debug("Got js: %r", js)
-        d = json.loads(str(js).decode("utf8"))
+        d = json.loads(js)
         start_time = update_counter("decode_json", start_time)
 
         logger.debug("Got internal_metadata: %r", internal_metadata)
-        internal_metadata = json.loads(str(internal_metadata).decode("utf8"))
+        internal_metadata = json.loads(internal_metadata)
         start_time = update_counter("decode_internal", start_time)
 
         ev = FrozenEvent(

synapse/storage/engines/maria.py

@@ -27,7 +27,7 @@ class MariaEngine(object):
 
     def encode_parameter(self, param):
         if isinstance(param, types.BufferType):
-            return str(param)
+            return bytes(param)
         return param
 
     def on_new_connection(self, db_conn):
@@ -45,3 +45,6 @@ class MariaEngine(object):
         if isinstance(error, self.module.DatabaseError):
             return error.sqlstate == "40001" and error.errno == 1213
         return False
+
+    def load_unicode(self, v):
+        return bytes(v).decode("UTF8")

synapse/storage/engines/sqlite3.py

@@ -15,6 +15,8 @@
 
 from synapse.storage import prepare_database, prepare_sqlite3_database
 
+import types
+
 
 class Sqlite3Engine(object):
     def __init__(self, database_module):
@@ -35,3 +37,8 @@ class Sqlite3Engine(object):
 
     def is_deadlock(self, error):
         return False
+
+    def load_unicode(self, v):
+        if isinstance(v, types.UnicodeType):
+            return v
+        return bytes(v).decode("UTF8")

synapse/storage/profile.py

@@ -36,7 +36,7 @@ class ProfileStore(SQLBaseStore):
         )
 
         if name:
-            name = name.decode("utf8")
+            name = self.database_engine.load_unicode(name)
 
         defer.returnValue(name)
 

synapse/storage/registration.py

@@ -99,7 +99,9 @@ class RegistrationStore(SQLBaseStore):
         )
 
         if user_info:
-            user_info["password_hash"] = user_info["password_hash"].decode("utf8")
+            user_info["password_hash"] = self.database_engine.load_unicode(
+                user_info["password_hash"]
+            )
 
         defer.returnValue(user_info)