From 37d766aeddf63450fc12fd2078bcaabe98042dd8 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 26 Oct 2017 09:59:50 +0100 Subject: Fix port script We changed _simple_update_one_txn to use _simple_update_txn but didn't yank it out in the port script. Fixes #2565 --- scripts/synapse_port_db | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index dc7fe940e8..a7a50e4d36 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -112,6 +112,7 @@ class Store(object): _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"] _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"] + _simple_update_txn = SQLBaseStore.__dict__["_simple_update_txn"] def runInteraction(self, desc, func, *args, **kwargs): def r(conn): -- cgit 1.4.1 From e86cefcb6f594bf66bde577899e996b5c75fc63f Mon Sep 17 00:00:00 2001 From: Luke Barnard Date: Thu, 26 Oct 2017 17:24:54 +0100 Subject: Add groups table to BOOLEAN_COLUMNS in synapse_port_db --- scripts/synapse_port_db | 1 + 1 file changed, 1 insertion(+) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index a7a50e4d36..d6d8ee50cb 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -42,6 +42,7 @@ BOOLEAN_COLUMNS = { "public_room_list_stream": ["visibility"], "device_lists_outbound_pokes": ["sent"], "users_who_share_rooms": ["share_private"], + "groups": ["is_public"], } -- cgit 1.4.1 From 9d419f48e635dbcc5ecc2e9fff7db85c320c0228 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 Oct 2017 16:58:49 +0000 Subject: Make the port script drop NUL values in all tables Postgres doesn't support NULs in strings so it makes the script throw an exception and stop if any values contain \0. Drop them with appropriate warning. --- scripts/synapse_port_db | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index d6d8ee50cb..3a8972efc3 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -320,7 +320,7 @@ class Porter(object): backward_chunk = min(row[0] for row in brows) - 1 rows = frows + brows - self._convert_rows(table, headers, rows) + rows = self._convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( @@ -556,17 +556,29 @@ class Porter(object): i for i, h in enumerate(headers) if h in bool_col_names ] + class BadValueException(Exception): + pass + def conv(j, col): if j in bool_cols: return bool(col) + elif isinstance(col, basestring) and "\0" in col: + logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col) + raise BadValueException(); return col + outrows = [] for i, row in enumerate(rows): - rows[i] = tuple( - conv(j, col) - for j, col in enumerate(row) - if j > 0 - ) + try: + outrows.append(tuple( + conv(j, col) + for j, col in enumerate(row) + if j > 0 + )) + except BadValueException: + pass + + return outrows @defer.inlineCallbacks def _setup_sent_transactions(self): @@ -594,7 +606,7 @@ class Porter(object): "select", r, ) - self._convert_rows("sent_transactions", headers, rows) + rows = self._convert_rows("sent_transactions", headers, rows) inserted_rows = len(rows) if inserted_rows: -- cgit 1.4.1 From 22db751d1edb6ce57c507b834cf892b45aaa51cd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Nov 2017 15:48:50 +0000 Subject: Add new boolean columns to port script --- scripts/synapse_port_db | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 3a8972efc3..0bebcdb6da 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -43,6 +43,13 @@ BOOLEAN_COLUMNS = { "device_lists_outbound_pokes": ["sent"], "users_who_share_rooms": ["share_private"], "groups": ["is_public"], + "groups_rooms": ["is_public"], + "groups_users": ["is_public", "is_admin"], + "group_summary_rooms": ["is_public"], + "group_room_categories": ["is_public"], + "group_summary_users": ["is_public"], + "group_roles": ["is_public"], + "local_group_membership": ["is_publicised", "is_admin"], } -- cgit 1.4.1 From 9ecf621404b8b961fc574bdc75df31f519cbb7c6 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 14 Nov 2017 15:55:15 +0000 Subject: Less s's --- scripts/synapse_port_db | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'scripts/synapse_port_db') diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index 0bebcdb6da..d46581e4e1 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -43,8 +43,8 @@ BOOLEAN_COLUMNS = { "device_lists_outbound_pokes": ["sent"], "users_who_share_rooms": ["share_private"], "groups": ["is_public"], - "groups_rooms": ["is_public"], - "groups_users": ["is_public", "is_admin"], + "group_rooms": ["is_public"], + "group_users": ["is_public", "is_admin"], "group_summary_rooms": ["is_public"], "group_room_categories": ["is_public"], "group_summary_users": ["is_public"], -- cgit 1.4.1