From 204132a998ec3be7069a9f2dada323bcdb217908 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:42:28 +0100 Subject: [PATCH 1/4] Check that postgres database has correct charset set --- scripts/port_from_sqlite_to_postgres.py | 4 ++++ synapse/app/homeserver.py | 11 ++++++++++- synapse/storage/engines/__init__.py | 1 + synapse/storage/engines/_base.py | 18 ++++++++++++++++++ synapse/storage/engines/postgres.py | 11 +++++++++++ synapse/storage/engines/sqlite3.py | 3 +++ 6 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/engines/_base.py diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py index 3296f1f54..da760af08 100644 --- a/scripts/port_from_sqlite_to_postgres.py +++ b/scripts/port_from_sqlite_to_postgres.py @@ -283,6 +283,10 @@ class Porter(object): self.sqlite_store = Store(sqlite_db_pool, sqlite_engine) self.postgres_store = Store(postgres_db_pool, postgres_engine) + yield self.postgres_store.execute( + postgres_engine.check_database + ) + # Step 1. Set up databases. self.progress.set_state("Preparing SQLite3") self.setup_db(sqlite_config, sqlite_engine) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 694a0125a..cbd295c0a 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -17,7 +17,7 @@ import sys sys.dont_write_bytecode = True -from synapse.storage.engines import create_engine +from synapse.storage.engines import create_engine, IncorrectDatabaseSetup from synapse.storage import ( are_all_users_on_domain, UpgradeDatabaseException, ) @@ -255,6 +255,15 @@ class SynapseHomeServer(HomeServer): ) sys.exit(1) + try: + database_engine.check_database(db_conn.cursor()) + except IncorrectDatabaseSetup as e: + sys.stderr.write("*" * len(e.message) + '\n') + sys.stderr.write(e.message) + sys.stderr.write('\n') + sys.stderr.write("*" * len(e.message) + '\n') + sys.exit(2) + def get_version_string(): try: diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index eb76df7f0..ab070f342 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from ._base import IncorrectDatabaseSetup from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py new file mode 100644 index 000000000..0b549d314 --- /dev/null +++ b/synapse/storage/engines/_base.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2015 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class IncorrectDatabaseSetup(RuntimeError): + pass diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index b8cca9b18..ca858daee 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -15,12 +15,23 @@ from synapse.storage import prepare_database +from ._base import IncorrectDatabaseSetup + class PostgresEngine(object): def __init__(self, database_module): self.module = database_module self.module.extensions.register_type(self.module.extensions.UNICODE) + def check_database(self, txn): + txn.execute("SHOW SERVER_ENCODING") + rows = txn.fetchall() + if rows and rows[0][0] != "UTF8": + raise IncorrectDatabaseSetup( + "Database has incorrect encoding: '%s' instead of 'UTF8'" + % (rows[0][0],) + ) + def convert_param_style(self, sql): return sql.replace("?", "%s") diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py index f62d5d120..7b49157cb 100644 --- a/synapse/storage/engines/sqlite3.py +++ b/synapse/storage/engines/sqlite3.py @@ -20,6 +20,9 @@ class Sqlite3Engine(object): def __init__(self, database_module): self.module = database_module + def check_database(self, txn): + pass + def convert_param_style(self, sql): return sql From a08bf11138b8adc37fcc80e4884e11e5546abcb5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:44:48 +0100 Subject: [PATCH 2/4] Appease PEP8 --- synapse/storage/engines/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index ab070f342..f661e4803 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,6 +14,8 @@ # limitations under the License. from ._base import IncorrectDatabaseSetup +assert IncorrectDatabaseSetup # Appease PEP8 + from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine From 4932a7e2d960be175c98b3766c14d52a0f2a7f3f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 11:56:38 +0100 Subject: [PATCH 3/4] Use __all__ instead of assert to stop pyflakes from warning about unused import in __init__ --- synapse/storage/engines/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py index f661e4803..bd3c8f945 100644 --- a/synapse/storage/engines/__init__.py +++ b/synapse/storage/engines/__init__.py @@ -14,8 +14,6 @@ # limitations under the License. from ._base import IncorrectDatabaseSetup -assert IncorrectDatabaseSetup # Appease PEP8 - from .postgres import PostgresEngine from .sqlite3 import Sqlite3Engine @@ -38,3 +36,6 @@ def create_engine(name): raise RuntimeError( "Unsupported database engine '%s'" % (name,) ) + + +__all__ = ["create_engine", "IncorrectDatabaseSetup"] From cd0864121bce3e1ab41ce432670cc87c4ffd54cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 29 Apr 2015 12:12:18 +0100 Subject: [PATCH 4/4] Make postgres database error slightly more helpful --- synapse/app/homeserver.py | 27 +++++++++++++++------------ synapse/storage/engines/postgres.py | 3 ++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index cbd295c0a..29a1bf1d7 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -245,24 +245,27 @@ class SynapseHomeServer(HomeServer): db_conn.cursor(), database_engine, self.hostname ) if not all_users_native: - sys.stderr.write( - "\n" - "******************************************************\n" + quit_with_error( "Found users in database not native to %s!\n" - "You cannot changed a synapse server_name after it's been configured\n" - "******************************************************\n" - "\n" % (self.hostname,) + "You cannot changed a synapse server_name after it's been configured" + % (self.hostname,) ) - sys.exit(1) try: database_engine.check_database(db_conn.cursor()) except IncorrectDatabaseSetup as e: - sys.stderr.write("*" * len(e.message) + '\n') - sys.stderr.write(e.message) - sys.stderr.write('\n') - sys.stderr.write("*" * len(e.message) + '\n') - sys.exit(2) + quit_with_error(e.message) + + +def quit_with_error(error_string): + message_lines = error_string.split("\n") + line_length = max([len(l) for l in message_lines]) + 2 + sys.stderr.write("*" * line_length + '\n') + for line in message_lines: + if line.strip(): + sys.stderr.write(" %s\n" % (line.strip(),)) + sys.stderr.write("*" * line_length + '\n') + sys.exit(1) def get_version_string(): diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index ca858daee..7125f66f0 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -28,7 +28,8 @@ class PostgresEngine(object): rows = txn.fetchall() if rows and rows[0][0] != "UTF8": raise IncorrectDatabaseSetup( - "Database has incorrect encoding: '%s' instead of 'UTF8'" + "Database has incorrect encoding: '%s' instead of 'UTF8'\n" + "See docs/postgres.rst for more information." % (rows[0][0],) )