Fix unicode support

22 files changed, 193 insertions, 168 deletions

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 1ab6effd5f..9d7a58080d 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -18,7 +18,7 @@ import sys
 sys.dont_write_bytecode = True
 
 from synapse.storage import (
-    prepare_database, prepare_sqlite3_database, UpgradeDatabaseException,
+    prepare_database, UpgradeDatabaseException,
 )
 from synapse.storage.engines import create_engine
 
@@ -381,19 +381,18 @@ def setup(config_options):
             "sql_mode": "TRADITIONAL",
             "charset": "utf8mb4",
             "use_unicode": True,
+            "collation": "utf8mb4_general_ci",
         })
     elif name == "sqlite3":
-        def open_fun(conn):
-            prepare_database(conn, database_engine)
         db_config.setdefault("args", {}).update({
             "cp_min": 1,
             "cp_max": 1,
-            "cp_openfun": open_fun,
         })
     else:
         raise RuntimeError("Unsupported database type '%s'" % (name,))
 
     database_engine = create_engine(name)
+    db_config["args"]["cp_openfun"] = database_engine.on_new_connection
 
     hs = SynapseHomeServer(
         config.server_name,
@@ -424,10 +423,7 @@ def setup(config_options):
             }
         )
 
-        if name == "sqlite3":
-            prepare_sqlite3_database(db_conn)
-
-        prepare_database(db_conn, database_engine)
+        database_engine.prepare_database(db_conn)
 
         db_conn.commit()
     except UpgradeDatabaseException:
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 047d100f46..de4f661973 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -142,19 +142,23 @@ class LoggingTransaction(object):
 
         sql = self.database_engine.convert_param_style(sql)
 
-        try:
-            if args and args[0]:
-                values = args[0]
+        if args and args[0]:
+            args = list(args)
+            args[0] = [
+                self.database_engine.encode_parameter(a) for a in args[0]
+            ]
+            try:
                 sql_logger.debug(
-                    "[SQL values] {%s} " + ", ".join(("<%r>",) * len(values)),
+                    "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])),
                     self.name,
-                    *values
+                    *args[0]
                 )
-        except:
-            # Don't let logging failures stop SQL from working
-            pass
+            except:
+                # Don't let logging failures stop SQL from working
+                pass
 
         start = time.time() * 1000
+
         try:
             return self.txn.execute(
                 sql, *args, **kwargs
@@ -761,8 +765,6 @@ class SQLBaseStore(object):
             return None
 
         internal_metadata, js, redacted, rejected_reason = res
-        js = js.decode("utf8")
-        internal_metadata = internal_metadata.decode("utf8")
 
         start_time = update_counter("select_event", start_time)
 
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 709b6f88ac..29702be923 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -16,6 +16,8 @@
 from .maria import MariaEngine
 from .sqlite3 import Sqlite3Engine
 
+import importlib
+
 
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
@@ -27,7 +29,7 @@ def create_engine(name):
     engine_class = SUPPORTED_MODULE.get(name, None)
 
     if engine_class:
-        module = __import__(name)
+        module = importlib.import_module(name)
         return engine_class(module)
 
     raise RuntimeError(
diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py
index df47763647..7fcb706a60 100644
--- a/synapse/storage/engines/maria.py
+++ b/synapse/storage/engines/maria.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.storage import prepare_database
 
 import types
 
@@ -28,3 +29,14 @@ class MariaEngine(object):
         if isinstance(param, types.BufferType):
             return str(param)
         return param
+
+    def on_new_connection(self, db_conn):
+        pass
+
+    def prepare_database(self, db_conn):
+        cur = db_conn.cursor()
+        cur.execute(
+            "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"
+        )
+        db_conn.commit()
+        prepare_database(db_conn, self)
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 639cdea41d..e802b5d5fd 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.storage import prepare_database, prepare_sqlite3_database
+
 
 class Sqlite3Engine(object):
     def __init__(self, database_module):
@@ -23,3 +25,10 @@ class Sqlite3Engine(object):
 
     def encode_parameter(self, param):
         return param
+
+    def on_new_connection(self, db_conn):
+        self.prepare_database(db_conn)
+
+    def prepare_database(self, db_conn):
+        prepare_sqlite3_database(db_conn)
+        prepare_database(db_conn, self)
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 25fef79434..d3b9b38664 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -57,15 +57,18 @@ class KeyStore(SQLBaseStore):
             OpenSSL.crypto.FILETYPE_ASN1, tls_certificate
         )
         fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest()
-        return self._simple_insert(
+        return self._simple_upsert(
             table="server_tls_certificates",
-            values={
+            keyvalues={
                 "server_name": server_name,
                 "fingerprint": fingerprint,
+            },
+            values={
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
                 "tls_certificate": buffer(tls_certificate_bytes),
             },
+            desc="store_server_certificate",
         )
 
     @defer.inlineCallbacks
@@ -106,13 +109,16 @@ class KeyStore(SQLBaseStore):
             ts_now_ms (int): The time now in milliseconds
             verification_key (VerifyKey): The NACL verify key.
         """
-        return self._simple_insert(
+        return self._simple_upsert(
             table="server_signature_keys",
-            values={
+            keyvalues={
                 "server_name": server_name,
                 "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
+            },
+            values={
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
                 "verify_key": buffer(verify_key.encode()),
             },
+            desc="store_server_verify_key",
         )
diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 90ac474859..717d289f78 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -14,16 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS rejections(
-    event_id VARCHAR(255) NOT NULL,
-    reason VARCHAR(255) NOT NULL,
-    last_check VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    reason VARCHAR(150) NOT NULL,
+    last_check VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
   app_id VARCHAR(64) NOT NULL,
@@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers (
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
-  rule_id VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
   priority_class TINYINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
-  conditions VARCHAR(255) NOT NULL,
-  actions VARCHAR(255) NOT NULL,
+  conditions VARCHAR(150) NOT NULL,
+  actions VARCHAR(150) NOT NULL,
   UNIQUE(user_name, rule_id)
 ) ;
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
-  user_id VARCHAR(255),
+  user_id VARCHAR(150),
   filter_id BIGINT,
   filter_json BLOB
 ) ;
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 4953b6323c..f5275a59b6 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -15,10 +15,10 @@
 
 CREATE TABLE IF NOT EXISTS application_services(
     id BIGINT PRIMARY KEY,
-    url VARCHAR(255),
-    token VARCHAR(255),
-    hs_token VARCHAR(255),
-    sender VARCHAR(255),
+    url VARCHAR(150),
+    token VARCHAR(150),
+    hs_token VARCHAR(150),
+    sender VARCHAR(150),
     UNIQUE(token)
 ) ;
 
@@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     id BIGINT PRIMARY KEY,
     as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex VARCHAR(255),
+    regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
 ) ;
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index 3bda073c94..1d582cc626 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,7 +1,7 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
-  rule_id VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 336cd563df..124c9a9bdf 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
-    event_id VARCHAR(255) NOT NULL,
-    prev_event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    prev_event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 ) ;
@@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
-    room_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
 ) ;
@@ -55,8 +55,8 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
-    event_id VARCHAR(255) NOT NULL,
-    destination VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    destination VARCHAR(150) NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 ) ;
@@ -65,10 +65,10 @@ CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id)
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -79,9 +79,9 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
-    event_id VARCHAR(255) NOT NULL,
-    auth_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    auth_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 11e611598b..30e3f71c5f 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
 ) ;
@@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
 ) ;
@@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(e
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id VARCHAR(255),
-    signature_name VARCHAR(255),
-    key_id VARCHAR(255),
+    event_id VARCHAR(150),
+    signature_name VARCHAR(150),
+    key_id VARCHAR(150),
     signature BLOB,
     UNIQUE (event_id, signature_name, key_id)
 ) ;
@@ -45,9 +45,9 @@ CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id VARCHAR(255),
-    prev_event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    prev_event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index a0fb337629..7cb8f802e1 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -16,9 +16,9 @@
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering BIGINT PRIMARY KEY,
     topological_ordering BIGINT NOT NULL,
-    event_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     content BLOB NOT NULL,
     unrecognized_keys BLOB,
     processed BOOL NOT NULL,
@@ -33,8 +33,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
     UNIQUE (event_id)
@@ -44,11 +44,11 @@ CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
-    prev_state VARCHAR(255),
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    prev_state VARCHAR(150),
     UNIQUE (event_id)
 ) ;
 
@@ -58,10 +58,10 @@ CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
 ) ;
@@ -71,11 +71,11 @@ CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (ty
 CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id VARCHAR(255) NOT NULL,
-    user_id VARCHAR(255) NOT NULL,
-    sender VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    membership VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
+    sender VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    membership VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
@@ -83,41 +83,41 @@ CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
-    event_id VARCHAR(255) NOT NULL,
-    feedback_type VARCHAR(255),
-    target_event_id VARCHAR(255),
-    sender VARCHAR(255),
-    room_id VARCHAR(255),
+    event_id VARCHAR(150) NOT NULL,
+    feedback_type VARCHAR(150),
+    target_event_id VARCHAR(150),
+    sender VARCHAR(150),
+    room_id VARCHAR(150),
     UNIQUE (event_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS topics(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    topic VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    topic VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    name VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    name VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 );
 
 CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
-    room_id VARCHAR(255) PRIMARY KEY NOT NULL,
+    room_id VARCHAR(150) PRIMARY KEY NOT NULL,
     is_public BOOL,
-    creator VARCHAR(255)
+    creator VARCHAR(150)
 ) ;
 
 CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id VARCHAR(255) NOT NULL,
-    host VARCHAR(255) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    host VARCHAR(150) NOT NULL,
     UNIQUE (room_id, host)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index a785cdb4c5..062ca53fef 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -13,18 +13,18 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name VARCHAR(255), -- Server name.
-  fingerprint VARCHAR(255), -- Certificate fingerprint.
-  from_server VARCHAR(255), -- Which key server the certificate was fetched from.
+  server_name VARCHAR(150), -- Server name.
+  fingerprint VARCHAR(150), -- Certificate fingerprint.
+  from_server VARCHAR(150), -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 ) ;
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name VARCHAR(255), -- Server name.
-  key_id VARCHAR(255), -- Key version.
-  from_server VARCHAR(255), -- Which key server the key was fetched form.
+  server_name VARCHAR(150), -- Server name.
+  key_id VARCHAR(150), -- Key version.
+  from_server VARCHAR(150), -- Which key server the key was fetched form.
   ts_added_ms BIGINT, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 27fe297af6..c8c5f1d2f0 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -14,21 +14,21 @@
  */
 
 CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id VARCHAR(255), -- The id used to refer to the media.
-    media_type VARCHAR(255), -- The MIME-type of the media.
+    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(255), -- The name the media was uploaded with.
-    user_id VARCHAR(255), -- The user who uploaded the file.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id VARCHAR(255), -- The id used to refer to the media.
+    media_id VARCHAR(150), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
-    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail.
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
@@ -39,25 +39,25 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin VARCHAR(255), -- The remote HS the media came from.
-    media_id VARCHAR(255), -- The id used to refer to the media on that server.
-    media_type VARCHAR(255), -- The MIME-type of the media.
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media on that server.
+    media_type VARCHAR(150), -- The MIME-type of the media.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(255), -- The name the media was uploaded with.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin VARCHAR(255), -- The remote HS the media came from.
-    media_id VARCHAR(255), -- The id used to refer to the media.
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail
-    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
     UNIQUE (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index b48b110ae9..273e61281a 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -13,9 +13,9 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS presence(
-  user_id VARCHAR(255) NOT NULL,
+  user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
-  status_msg VARCHAR(255),
+  status_msg VARCHAR(150),
   mtime BIGINT, -- miliseconds since last state change
   UNIQUE(user_id)
 ) ;
@@ -23,16 +23,14 @@ CREATE TABLE IF NOT EXISTS presence(
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id VARCHAR(255) NOT NULL,
-  observer_user_id VARCHAR(255), -- a UserID,
-  UNIQUE(observed_user_id)
+  observed_user_id VARCHAR(150) NOT NULL,
+  observer_user_id VARCHAR(150) -- a UserID,
 ) ;
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
-  user_id VARCHAR(255) NOT NULL,
-  observed_user_id VARCHAR(255), -- a UserID,
-  accepted BOOLEAN,
-  UNIQUE(user_id)
+  user_id VARCHAR(150) NOT NULL,
+  observed_user_id VARCHAR(150), -- a UserID,
+  accepted BOOLEAN
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 92da48f97e..023060a54c 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS profiles(
-    user_id VARCHAR(255) NOT NULL,
-    displayname VARCHAR(255),
-    avatar_url VARCHAR(255),
+    user_id VARCHAR(150) NOT NULL,
+    displayname VARCHAR(150),
+    avatar_url VARCHAR(150),
     UNIQUE(user_id)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 9b52a2012a..5c23188d62 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS redactions (
-    event_id VARCHAR(255) NOT NULL,
-    redacts VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    redacts VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 220df87573..63fe0f5c64 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -14,12 +14,12 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias VARBINARY(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    room_alias VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (room_alias)
 ) ;
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias VARBINARY(255) NOT NULL,
-    server VARCHAR(255) NOT NULL
+    room_alias VARCHAR(150) NOT NULL,
+    server VARCHAR(150) NOT NULL
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 40584a325f..acfb76439b 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -15,21 +15,21 @@
 
 CREATE TABLE IF NOT EXISTS state_groups(
     id VARCHAR(20) PRIMARY KEY,
-    room_id VARCHAR(255) NOT NULL,
-    event_id VARCHAR(255) NOT NULL
+    room_id VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
 ) ;
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group VARCHAR(20) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
-    event_id VARCHAR(255) NOT NULL
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
 ) ;
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id VARCHAR(255) NOT NULL,
-    state_group VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    state_group VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index d33bdfb301..43541661ce 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -14,8 +14,8 @@
  */
 -- Stores what transaction ids we have received and what our response was
 CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id VARCHAR(255),
-    origin VARCHAR(255),
+    transaction_id VARCHAR(150),
+    origin VARCHAR(150),
     ts BIGINT,
     response_code INTEGER,
     response_json BLOB,
@@ -30,8 +30,8 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
     id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
-    transaction_id VARCHAR(255),
-    destination VARCHAR(255),
+    transaction_id VARCHAR(150),
+    destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
     ts BIGINT
@@ -47,9 +47,9 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c
 -- For sent transactions only.
 CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
-    destination VARCHAR(255),
-    pdu_id VARCHAR(255),
-    pdu_origin VARCHAR(255),
+    destination VARCHAR(150),
+    pdu_id VARCHAR(150),
+    pdu_origin VARCHAR(150),
     UNIQUE (transaction_id, destination)
 ) ;
 
@@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
-    destination VARCHAR(255) PRIMARY KEY,
+    destination VARCHAR(150) PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 28909f5805..0271de3526 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    name VARCHAR(255),
-    password_hash VARCHAR(255),
+    name VARCHAR(150),
+    password_hash VARCHAR(150),
     creation_ts BIGINT,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
@@ -22,19 +22,19 @@ CREATE TABLE IF NOT EXISTS users(
 
 CREATE TABLE IF NOT EXISTS access_tokens(
     id BIGINT PRIMARY KEY,
-    user_id VARCHAR(255) NOT NULL,
-    device_id VARCHAR(255),
-    token VARCHAR(255) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    token VARCHAR(150) NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
 ) ;
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user VARCHAR(255) NOT NULL,
-    access_token VARCHAR(255) NOT NULL,
-    device_id VARCHAR(255),
-    ip VARCHAR(255) NOT NULL,
-    user_agent VARCHAR(255) NOT NULL,
+    user VARCHAR(150) NOT NULL,
+    access_token VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    ip VARCHAR(150) NOT NULL,
+    user_agent VARCHAR(150) NOT NULL,
     last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
 ) ;
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index 28762861e5..e7fa6fe569 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS schema_version(
 
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
     `version` INTEGER NOT NULL,
-    `file` VARCHAR(255) NOT NULL,
+    `file` VARCHAR(150) NOT NULL,
     UNIQUE(version, file)
 );