diff options
| author | Richard van der Hoff <github@rvanderhoff.org.uk> | 2017-10-31 17:16:08 +0000 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2017-10-31 17:16:08 +0000 |
| commit | e025aec028741702e9325264a83e027c0b6ab3bd (patch) | |
| tree | 88a8e0b085ffdd32461387b98646adfbcd5fe6b9 | |
| parent | Merge pull request #2609 from matrix-org/rav/refactor_login (diff) | |
| parent | Make the port script drop NUL values in all tables (diff) | |
| download | synapse-e025aec028741702e9325264a83e027c0b6ab3bd.tar.xz | |
Merge pull request #2611 from matrix-org/dbkr/port_script_drop_nuls
Make the port script drop NUL values in all tables
| -rwxr-xr-x | scripts/synapse_port_db | 26 |
1 files changed, 19 insertions, 7 deletions
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index d6d8ee50cb..3a8972efc3 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -320,7 +320,7 @@ class Porter(object): backward_chunk = min(row[0] for row in brows) - 1 rows = frows + brows - self._convert_rows(table, headers, rows) + rows = self._convert_rows(table, headers, rows) def insert(txn): self.postgres_store.insert_many_txn( @@ -556,17 +556,29 @@ class Porter(object): i for i, h in enumerate(headers) if h in bool_col_names ] + class BadValueException(Exception): + pass + def conv(j, col): if j in bool_cols: return bool(col) + elif isinstance(col, basestring) and "\0" in col: + logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col) + raise BadValueException(); return col + outrows = [] for i, row in enumerate(rows): - rows[i] = tuple( - conv(j, col) - for j, col in enumerate(row) - if j > 0 - ) + try: + outrows.append(tuple( + conv(j, col) + for j, col in enumerate(row) + if j > 0 + )) + except BadValueException: + pass + + return outrows @defer.inlineCallbacks def _setup_sent_transactions(self): @@ -594,7 +606,7 @@ class Porter(object): "select", r, ) - self._convert_rows("sent_transactions", headers, rows) + rows = self._convert_rows("sent_transactions", headers, rows) inserted_rows = len(rows) if inserted_rows: |