From 6482075c95957ad980d9c1323f9f982e6f7aaff4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 14 May 2021 11:46:35 +0100 Subject: Run `black` on the scripts (#9981) Turns out these scripts weren't getting linted. --- scripts/synapse_port_db | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 5fb5bb35f7..7c7645c05a 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -294,8 +294,7 @@ class Porter(object): return table, already_ported, total_to_port, forward_chunk, backward_chunk async def get_table_constraints(self) -> Dict[str, Set[str]]: - """Returns a map of tables that have foreign key constraints to tables they depend on. - """ + """Returns a map of tables that have foreign key constraints to tables they depend on.""" def _get_constraints(txn): # We can pull the information about foreign key constraints out from @@ -504,7 +503,9 @@ class Porter(object): return def build_db_store( - self, db_config: DatabaseConnectionConfig, allow_outdated_version: bool = False, + self, + db_config: DatabaseConnectionConfig, + allow_outdated_version: bool = False, ): """Builds and returns a database store using the provided configuration. @@ -740,7 +741,7 @@ class Porter(object): return col outrows = [] - for i, row in enumerate(rows): + for row in rows: try: outrows.append( tuple(conv(j, col) for j, col in enumerate(row) if j > 0) @@ -890,8 +891,7 @@ class Porter(object): await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r) async def _setup_events_stream_seqs(self) -> None: - """Set the event stream sequences to the correct values. - """ + """Set the event stream sequences to the correct values.""" # We get called before we've ported the events table, so we need to # fetch the current positions from the SQLite store. @@ -920,12 +920,14 @@ class Porter(object): ) await self.postgres_store.db_pool.runInteraction( - "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos, + "_setup_events_stream_seqs", + _setup_events_stream_seqs_set_pos, ) - async def _setup_sequence(self, sequence_name: str, stream_id_tables: Iterable[str]) -> None: - """Set a sequence to the correct value. - """ + async def _setup_sequence( + self, sequence_name: str, stream_id_tables: Iterable[str] + ) -> None: + """Set a sequence to the correct value.""" current_stream_ids = [] for stream_id_table in stream_id_tables: max_stream_id = await self.sqlite_store.db_pool.simple_select_one_onecol( @@ -939,14 +941,19 @@ class Porter(object): next_id = max(current_stream_ids) + 1 def r(txn): - sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name, ) - txn.execute(sql + " %s", (next_id, )) + sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,) + txn.execute(sql + " %s", (next_id,)) - await self.postgres_store.db_pool.runInteraction("_setup_%s" % (sequence_name,), r) + await self.postgres_store.db_pool.runInteraction( + "_setup_%s" % (sequence_name,), r + ) async def _setup_auth_chain_sequence(self) -> None: curr_chain_id = await self.sqlite_store.db_pool.simple_select_one_onecol( - table="event_auth_chains", keyvalues={}, retcol="MAX(chain_id)", allow_none=True + table="event_auth_chains", + keyvalues={}, + retcol="MAX(chain_id)", + allow_none=True, ) def r(txn): @@ -968,8 +975,7 @@ class Porter(object): class Progress(object): - """Used to report progress of the port - """ + """Used to report progress of the port""" def __init__(self): self.tables = {} @@ -994,8 +1000,7 @@ class Progress(object): class CursesProgress(Progress): - """Reports progress to a curses window - """ + """Reports progress to a curses window""" def __init__(self, stdscr): self.stdscr = stdscr @@ -1020,7 +1025,7 @@ class CursesProgress(Progress): self.total_processed = 0 self.total_remaining = 0 - for table, data in self.tables.items(): + for data in self.tables.values(): self.total_processed += data["num_done"] - data["start"] self.total_remaining += data["total"] - data["num_done"] @@ -1111,8 +1116,7 @@ class CursesProgress(Progress): class TerminalProgress(Progress): - """Just prints progress to the terminal - """ + """Just prints progress to the terminal""" def update(self, table, num_done): super(TerminalProgress, self).update(table, num_done) -- cgit 1.5.1 From daca7b2794fb86514dc01de551bb0e5db77cf914 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 24 May 2021 14:03:00 +0100 Subject: Fix off-by-one-error in synapse_port_db (#9991) fixes #9979 --- .buildkite/postgres-config.yaml | 6 ++---- .buildkite/scripts/test_synapse_port_db.sh | 4 ++++ .buildkite/sqlite-config.yaml | 6 ++---- changelog.d/9991.bugfix | 1 + scripts/synapse_port_db | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 changelog.d/9991.bugfix (limited to 'scripts/synapse_port_db') diff --git a/.buildkite/postgres-config.yaml b/.buildkite/postgres-config.yaml index 2acbe66f4c..67e17fa9d1 100644 --- a/.buildkite/postgres-config.yaml +++ b/.buildkite/postgres-config.yaml @@ -3,7 +3,7 @@ # CI's Docker setup at the point where this file is considered. server_name: "localhost:8800" -signing_key_path: "/src/.buildkite/test.signing.key" +signing_key_path: ".buildkite/test.signing.key" report_stats: false @@ -16,6 +16,4 @@ database: database: synapse # Suppress the key server warning. -trusted_key_servers: - - server_name: "matrix.org" -suppress_key_server_warning: true +trusted_key_servers: [] diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh index a7e2454769..82d7d56d4e 100755 --- a/.buildkite/scripts/test_synapse_port_db.sh +++ b/.buildkite/scripts/test_synapse_port_db.sh @@ -33,6 +33,10 @@ scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml echo "+++ Run synapse_port_db against test database" coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml +# We should be able to run twice against the same database. +echo "+++ Run synapse_port_db a second time" +coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml + ##### # Now do the same again, on an empty database. diff --git a/.buildkite/sqlite-config.yaml b/.buildkite/sqlite-config.yaml index 6d9bf80d84..d16459cfd9 100644 --- a/.buildkite/sqlite-config.yaml +++ b/.buildkite/sqlite-config.yaml @@ -3,7 +3,7 @@ # schema and run background updates on it. server_name: "localhost:8800" -signing_key_path: "/src/.buildkite/test.signing.key" +signing_key_path: ".buildkite/test.signing.key" report_stats: false @@ -13,6 +13,4 @@ database: database: ".buildkite/test_db.db" # Suppress the key server warning. -trusted_key_servers: - - server_name: "matrix.org" -suppress_key_server_warning: true +trusted_key_servers: [] diff --git a/changelog.d/9991.bugfix b/changelog.d/9991.bugfix new file mode 100644 index 0000000000..665ff04dea --- /dev/null +++ b/changelog.d/9991.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in v1.26.0 which meant that `synapse_port_db` would not correctly initialise some postgres sequences, requiring manual updates afterwards. diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 7c7645c05a..86eb76cbca 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -959,7 +959,7 @@ class Porter(object): def r(txn): txn.execute( "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s", - (curr_chain_id,), + (curr_chain_id + 1,), ) if curr_chain_id is not None: -- cgit 1.5.1