From e01ba5bda315ed80f2a809e2f2e917eaf7488fac Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 6 Apr 2018 13:45:10 +0100 Subject: [PATCH 1/3] Port script: avoid nasty errors when setting up We really shouldn't spit out "Failed to create port table", it looks scary. --- scripts/synapse_port_db | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index d46581e4e..7fb8be3ab 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2018 New Vector Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -491,7 +492,7 @@ class Porter(object): def create_port_table(txn): txn.execute( - "CREATE TABLE port_from_sqlite3 (" + "CREATE TABLE IF NOT EXISTS port_from_sqlite3 (" " table_name varchar(100) NOT NULL UNIQUE," " forward_rowid bigint NOT NULL," " backward_rowid bigint NOT NULL" @@ -517,14 +518,11 @@ class Porter(object): "alter_table", alter_table ) except Exception as e: - logger.info("Failed to create port table: %s", e) + pass - try: - yield self.postgres_store.runInteraction( - "create_port_table", create_port_table - ) - except Exception as e: - logger.info("Failed to create port table: %s", e) + yield self.postgres_store.runInteraction( + "create_port_table", create_port_table + ) self.progress.set_state("Setting up") From 01579384ccd26d112df5169522fd16234e8316c7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 6 Apr 2018 13:47:11 +0100 Subject: [PATCH 2/3] Port script: clean up a bit Improve logging and comments. Group all the stuff to do with inspecting tables together rather than creating the port tables in the middle. --- scripts/synapse_port_db | 54 ++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 7fb8be3ab..dc1a10bf2 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -251,6 +251,12 @@ class Porter(object): @defer.inlineCallbacks def handle_table(self, table, postgres_size, table_size, forward_chunk, backward_chunk): + logger.info( + "Table %s: %i/%i (rows %i-%i) already ported", + table, postgres_size, table_size, + backward_chunk+1, forward_chunk-1, + ) + if not table_size: return @@ -468,28 +474,7 @@ class Porter(object): self.progress.set_state("Preparing PostgreSQL") self.setup_db(postgres_config, postgres_engine) - # Step 2. Get tables. - self.progress.set_state("Fetching tables") - sqlite_tables = yield self.sqlite_store._simple_select_onecol( - table="sqlite_master", - keyvalues={ - "type": "table", - }, - retcol="name", - ) - - postgres_tables = yield self.postgres_store._simple_select_onecol( - table="information_schema.tables", - keyvalues={}, - retcol="distinct table_name", - ) - - tables = set(sqlite_tables) & set(postgres_tables) - - self.progress.set_state("Creating tables") - - logger.info("Found %d tables", len(tables)) - + self.progress.set_state("Creating port tables") def create_port_table(txn): txn.execute( "CREATE TABLE IF NOT EXISTS port_from_sqlite3 (" @@ -524,9 +509,27 @@ class Porter(object): "create_port_table", create_port_table ) - self.progress.set_state("Setting up") + # Step 2. Get tables. + self.progress.set_state("Fetching tables") + sqlite_tables = yield self.sqlite_store._simple_select_onecol( + table="sqlite_master", + keyvalues={ + "type": "table", + }, + retcol="name", + ) - # Set up tables. + postgres_tables = yield self.postgres_store._simple_select_onecol( + table="information_schema.tables", + keyvalues={}, + retcol="distinct table_name", + ) + + tables = set(sqlite_tables) & set(postgres_tables) + logger.info("Found %d tables", len(tables)) + + # Step 3. Figure out what still needs copying + self.progress.set_state("Checking on port progress") setup_res = yield defer.gatherResults( [ self.setup_table(table) @@ -537,7 +540,8 @@ class Porter(object): consumeErrors=True, ) - # Process tables. + # Step 4. Do the copying. + self.progress.set_state("Copying to postgres") yield defer.gatherResults( [ self.handle_table(*res) From 6a9777ba028af1d9803d13e879eaf62773c5bd83 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 6 Apr 2018 13:48:40 +0100 Subject: [PATCH 3/3] Port script: Set up state_group_id_seq Fixes https://github.com/matrix-org/synapse/issues/3050. --- scripts/synapse_port_db | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index dc1a10bf2..7b23a4485 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -550,6 +550,9 @@ class Porter(object): consumeErrors=True, ) + # Step 5. Do final post-processing + yield self._setup_state_group_id_seq() + self.progress.done() except: global end_error_exec_info @@ -709,6 +712,16 @@ class Porter(object): defer.returnValue((done, remaining + done)) + def _setup_state_group_id_seq(self): + def r(txn): + txn.execute("SELECT MAX(id) FROM state_groups") + next_id = txn.fetchone()[0]+1 + txn.execute( + "ALTER SEQUENCE state_group_id_seq RESTART WITH %s", + (next_id,), + ) + return self.postgres_store.runInteraction("setup_state_group_id_seq", r) + ############################################## ###### The following is simply UI stuff ######