Support running porting script multiple times

This commit is contained in:
Erik Johnston 2015-04-13 13:45:04 +01:00
parent 1bede47843
commit 90bcb86957

View file

@ -51,16 +51,52 @@ UNICODE_COLUMNS = {
} }
APPEND_ONLY_TABLES = [
"event_content_hashes",
"event_reference_hashes",
"event_signatures",
"event_edge_hashes",
"events",
"event_json",
"state_events",
"room_memberships",
"feedback",
"topics",
"room_names",
"rooms",
"local_media_repository",
"local_media_repository_thumbnails",
"remote_media_cache",
"remote_media_cache_thumbnails",
"redactions",
"event_edges",
"event_auth",
"received_transactions",
"sent_transactions",
"transaction_id_to_pdu",
"users",
"state_groups",
"state_groups_state",
"event_to_state_groups",
"rejections",
]
class Store(object): class Store(object):
def __init__(self, db_pool, engine): def __init__(self, db_pool, engine):
self.db_pool = db_pool self.db_pool = db_pool
self.engine = engine self.database_engine = engine
_simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"] _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
_simple_insert = SQLBaseStore.__dict__["_simple_insert"] _simple_insert = SQLBaseStore.__dict__["_simple_insert"]
_simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"] _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
_simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"] _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
_simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
_simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"]
_simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
_simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
_execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"] _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"]
@ -73,11 +109,11 @@ class Store(object):
try: try:
txn = conn.cursor() txn = conn.cursor()
return func( return func(
LoggingTransaction(txn, desc, self.engine), LoggingTransaction(txn, desc, self.database_engine),
*args, **kwargs *args, **kwargs
) )
except self.engine.module.DatabaseError as e: except self.database_engine.module.DatabaseError as e:
if self.engine.is_deadlock(e): if self.database_engine.is_deadlock(e):
logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N) logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N)
if i < N: if i < N:
i += 1 i += 1
@ -117,16 +153,50 @@ def chunks(n):
@defer.inlineCallbacks @defer.inlineCallbacks
def handle_table(table, sqlite_store, mysql_store): def handle_table(table, sqlite_store, mysql_store):
N = 1000 if table in APPEND_ONLY_TABLES:
# It's safe to just carry on inserting.
next_chunk = yield mysql_store._simple_select_one_onecol(
table="port_from_sqlite3",
keyvalues={"table_name": table},
retcol="rowid",
allow_none=True,
)
if next_chunk is None:
yield mysql_store._simple_insert(
table="port_from_sqlite3",
values={"table_name": table, "rowid": 0}
)
next_chunk = 0
else:
def delete_all(txn):
txn.execute(
"DELETE FROM port_from_sqlite3 WHERE table_name = %s",
(table,)
)
txn.execute("DELETE FROM %s" % (table,))
mysql_store._simple_insert_txn(
txn,
table="port_from_sqlite3",
values={"table_name": table, "rowid": 0}
)
yield mysql_store.runInteraction(
"delete_non_append_only", delete_all
)
next_chunk = 0
N = 5000
select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,) select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
uni_col_names = UNICODE_COLUMNS.get(table, []) uni_col_names = UNICODE_COLUMNS.get(table, [])
def conv_uni(c): def conv_uni(c):
return sqlite_store.engine.load_unicode(c) return sqlite_store.database_engine.load_unicode(c)
next_chunk = 0
while True: while True:
def r(txn): def r(txn):
txn.execute(select, (next_chunk, N,)) txn.execute(select, (next_chunk, N,))
@ -145,7 +215,7 @@ def handle_table(table, sqlite_store, mysql_store):
for i, row in enumerate(rows): for i, row in enumerate(rows):
rows[i] = tuple( rows[i] = tuple(
mysql_store.engine.encode_parameter( mysql_store.database_engine.encode_parameter(
conv_uni(col) if j in uni_cols else col conv_uni(col) if j in uni_cols else col
) )
for j, col in enumerate(row) for j, col in enumerate(row)
@ -153,6 +223,12 @@ def handle_table(table, sqlite_store, mysql_store):
) )
yield mysql_store.insert_many(table, headers[1:], rows) yield mysql_store.insert_many(table, headers[1:], rows)
yield mysql_store._simple_update_one(
table="port_from_sqlite3",
keyvalues={"table_name": table},
updatevalues={"rowid": next_chunk},
)
else: else:
return return
@ -208,6 +284,22 @@ def main(sqlite_config, mysql_config):
logger.info("Found %d tables", len(tables)) logger.info("Found %d tables", len(tables))
def create_port_table(txn):
try:
txn.execute(
"CREATE TABLE port_from_sqlite3 ("
" `table_name` varchar(100) NOT NULL UNIQUE,"
" `rowid` bigint unsigned NOT NULL"
")"
)
except mysql_engine.module.DatabaseError as e:
if e.errno != mysql_engine.module.errorcode.ER_TABLE_EXISTS_ERROR:
raise
yield mysql_store.runInteraction(
"create_port_table", create_port_table
)
# Process tables. # Process tables.
yield defer.gatherResults( yield defer.gatherResults(
[ [