summary refs log tree commit diff
path: root/scripts/synapse_port_db
diff options
context:
space:
mode:
authorDavid Baker <dave@matrix.org>2017-10-31 16:58:49 +0000
committerDavid Baker <dave@matrix.org>2017-10-31 16:58:49 +0000
commit9d419f48e635dbcc5ecc2e9fff7db85c320c0228 (patch)
tree88a8e0b085ffdd32461387b98646adfbcd5fe6b9 /scripts/synapse_port_db
parentMerge pull request #2609 from matrix-org/rav/refactor_login (diff)
downloadsynapse-9d419f48e635dbcc5ecc2e9fff7db85c320c0228.tar.xz
Make the port script drop NUL values in all tables
Postgres doesn't support NULs in strings so it makes the script
throw an exception and stop if any values contain \0. Drop them
with appropriate warning.
Diffstat (limited to 'scripts/synapse_port_db')
-rwxr-xr-xscripts/synapse_port_db26
1 files changed, 19 insertions, 7 deletions
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index d6d8ee50cb..3a8972efc3 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -320,7 +320,7 @@ class Porter(object):
                     backward_chunk = min(row[0] for row in brows) - 1
 
                 rows = frows + brows
-                self._convert_rows(table, headers, rows)
+                rows = self._convert_rows(table, headers, rows)
 
                 def insert(txn):
                     self.postgres_store.insert_many_txn(
@@ -556,17 +556,29 @@ class Porter(object):
             i for i, h in enumerate(headers) if h in bool_col_names
         ]
 
+        class BadValueException(Exception):
+            pass
+
         def conv(j, col):
             if j in bool_cols:
                 return bool(col)
+            elif isinstance(col, basestring) and "\0" in col:
+                logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
+                raise BadValueException();
             return col
 
+        outrows = []
         for i, row in enumerate(rows):
-            rows[i] = tuple(
-                conv(j, col)
-                for j, col in enumerate(row)
-                if j > 0
-            )
+            try:
+                outrows.append(tuple(
+                    conv(j, col)
+                    for j, col in enumerate(row)
+                    if j > 0
+                ))
+            except BadValueException:
+                pass
+
+        return outrows
 
     @defer.inlineCallbacks
     def _setup_sent_transactions(self):
@@ -594,7 +606,7 @@ class Porter(object):
             "select", r,
         )
 
-        self._convert_rows("sent_transactions", headers, rows)
+        rows = self._convert_rows("sent_transactions", headers, rows)
 
         inserted_rows = len(rows)
         if inserted_rows: