From 89036579ed475464ccc8619344db513042e1d792 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 18 Mar 2015 11:18:49 +0000
Subject: Update schema to work with mariadb

---
 synapse/storage/schema/delta/12/v12.sql            | 48 +++++------
 synapse/storage/schema/delta/13/v13.sql            | 19 ++---
 synapse/storage/schema/delta/14/v14.sql            |  6 +-
 .../storage/schema/full_schemas/11/event_edges.sql | 52 ++++++------
 .../schema/full_schemas/11/event_signatures.sql    | 46 ++++------
 synapse/storage/schema/full_schemas/11/im.sql      | 98 +++++++++++-----------
 synapse/storage/schema/full_schemas/11/keys.sql    | 16 ++--
 .../schema/full_schemas/11/media_repository.sql    | 46 +++++-----
 .../storage/schema/full_schemas/11/presence.sql    | 18 ++--
 .../storage/schema/full_schemas/11/profiles.sql    |  8 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  6 +-
 .../schema/full_schemas/11/room_aliases.sql        | 12 ++-
 synapse/storage/schema/full_schemas/11/state.sql   | 29 +++----
 .../schema/full_schemas/11/transactions.sql        | 32 ++++---
 synapse/storage/schema/full_schemas/11/users.sql   | 29 ++++---
 synapse/storage/schema/schema_version.sql          | 15 ++--
 16 files changed, 224 insertions(+), 256 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index b87ef1fe79..fbe8486b96 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -14,54 +14,50 @@
  */
 
 CREATE TABLE IF NOT EXISTS rejections(
-    event_id TEXT NOT NULL,
-    reason TEXT NOT NULL,
-    last_check TEXT NOT NULL,
-    CONSTRAINT ev_id UNIQUE (event_id) ON CONFLICT REPLACE
+    event_id VARCHAR(255) NOT NULL,
+    reason VARCHAR(255) NOT NULL,
+    last_check VARCHAR(255) NOT NULL,
+    UNIQUE (event_id)
 );
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name TEXT NOT NULL,
-  profile_tag varchar(32) NOT NULL,
-  kind varchar(8) NOT NULL,
-  app_id varchar(64) NOT NULL,
-  app_display_name varchar(64) NOT NULL,
-  device_display_name varchar(128) NOT NULL,
-  pushkey blob NOT NULL,
+  id INTEGER PRIMARY KEY,
+  user_name VARCHAR(255) NOT NULL,
+  profile_tag VARCHAR(32) NOT NULL,
+  kind VARCHAR(8) NOT NULL,
+  app_id VARCHAR(64) NOT NULL,
+  app_display_name VARCHAR(64) NOT NULL,
+  device_display_name VARCHAR(128) NOT NULL,
+  pushkey VARBINARY(512) NOT NULL,
   ts BIGINT NOT NULL,
-  lang varchar(8),
-  data blob,
+  lang VARCHAR(8),
+  data BLOB,
   last_token TEXT,
   last_success BIGINT,
   failing_since BIGINT,
-  FOREIGN KEY(user_name) REFERENCES users(name),
   UNIQUE (app_id, pushkey)
 );
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name TEXT NOT NULL,
-  rule_id TEXT NOT NULL,
+  id INTEGER PRIMARY KEY,
+  user_name VARCHAR(255) NOT NULL,
+  rule_id VARCHAR(255) NOT NULL,
   priority_class TINYINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
-  conditions TEXT NOT NULL,
-  actions TEXT NOT NULL,
+  conditions VARCHAR(255) NOT NULL,
+  actions VARCHAR(255) NOT NULL,
   UNIQUE(user_name, rule_id)
 );
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
-  user_id TEXT,
+  user_id VARCHAR(255),
   filter_id INTEGER,
-  filter_json TEXT,
-  FOREIGN KEY(user_id) REFERENCES users(id)
+  filter_json BLOB
 );
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
-  user_id, filter_id
+    user_id, filter_id
 );
-
-PRAGMA user_version = 12;
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index e491ad5aec..68d4ebd896 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -14,21 +14,18 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    url TEXT,
-    token TEXT,
-    hs_token TEXT,
-    sender TEXT,
-    UNIQUE(token) ON CONFLICT ROLLBACK
+    id INTEGER PRIMARY KEY,
+    url VARCHAR(255),
+    token VARCHAR(255),
+    hs_token VARCHAR(255),
+    sender VARCHAR(255),
+    UNIQUE(token)
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY,
     as_id INTEGER NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex TEXT,
+    regex VARCHAR(255),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
 );
-
-
-
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index 0212726448..2b50adbd61 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,7 +1,7 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name TEXT NOT NULL,
-  rule_id TEXT NOT NULL,
+  id INTEGER PRIMARY KEY,
+  user_name VARCHAR(255) NOT NULL,
+  rule_id VARCHAR(255) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 1e766d6db2..823653f2e9 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -14,9 +14,9 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    UNIQUE (event_id, room_id)
 );
 
 CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
@@ -24,9 +24,9 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    UNIQUE (event_id, room_id)
 );
 
 CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
@@ -34,11 +34,11 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
-    event_id TEXT NOT NULL,
-    prev_event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    is_state INTEGER NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, prev_event_id, room_id, is_state)
+    event_id VARCHAR(255) NOT NULL,
+    prev_event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    is_state BOOL NOT NULL,
+    UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
 
 CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
@@ -46,30 +46,30 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
-    room_id TEXT NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
     min_depth INTEGER NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (room_id)
+    UNIQUE (room_id)
 );
 
 CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
-    event_id TEXT NOT NULL,
-    destination TEXT NOT NULL,
+    event_id VARCHAR(255) NOT NULL,
+    destination VARCHAR(255) NOT NULL,
     delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered
-    CONSTRAINT uniqueness UNIQUE (event_id, destination) ON CONFLICT REPLACE
+    UNIQUE (event_id, destination)
 );
 
 CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, room_id) ON CONFLICT REPLACE
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    state_key VARCHAR(255) NOT NULL,
+    UNIQUE (event_id, room_id)
 );
 
 CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
@@ -79,11 +79,11 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
-    event_id TEXT NOT NULL,
-    auth_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    CONSTRAINT uniqueness UNIQUE (event_id, auth_id, room_id)
+    event_id VARCHAR(255) NOT NULL,
+    auth_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    UNIQUE (event_id, auth_id, room_id)
 );
 
 CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
-CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
\ No newline at end of file
+CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index c28c39c48a..a7c3acf775 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -14,52 +14,42 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id TEXT,
-    algorithm TEXT,
+    event_id VARCHAR(255),
+    algorithm VARCHAR(255),
     hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
+    UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(
-    event_id
-);
+CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id TEXT,
-    algorithm TEXT,
+    event_id VARCHAR(255),
+    algorithm VARCHAR(255),
     hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, algorithm)
+    UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes (
-    event_id
-);
+CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id TEXT,
-    signature_name TEXT,
-    key_id TEXT,
+    event_id VARCHAR(255),
+    signature_name VARCHAR(255),
+    key_id VARCHAR(255),
     signature BLOB,
-    CONSTRAINT uniqueness UNIQUE (event_id, signature_name, key_id)
+    UNIQUE (event_id, signature_name, key_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures (
-    event_id
-);
+CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id TEXT,
-    prev_event_id TEXT,
-    algorithm TEXT,
+    event_id VARCHAR(255),
+    prev_event_id VARCHAR(255),
+    algorithm VARCHAR(255),
     hash BLOB,
-    CONSTRAINT uniqueness UNIQUE (
-        event_id, prev_event_id, algorithm
-    )
+    UNIQUE (event_id, prev_event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(
-    event_id
-);
+CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index dd00c1cd2f..8feacdbe08 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -14,17 +14,17 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
+    stream_ordering INTEGER PRIMARY KEY,
     topological_ordering INTEGER NOT NULL,
-    event_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    content TEXT NOT NULL,
-    unrecognized_keys TEXT,
+    event_id VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    content BLOB NOT NULL,
+    unrecognized_keys BLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth INTEGER DEFAULT 0 NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
+    UNIQUE (event_id)
 );
 
 CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
@@ -34,92 +34,92 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    internal_metadata NOT NULL,
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
-    CONSTRAINT ev_j_uniq UNIQUE (event_id)
+    UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_json_id ON event_json(event_id);
 CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
 
-CREATE TABLE IF NOT EXISTS state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    prev_state TEXT
+CREATE TABLE state_events(
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    state_key VARCHAR(255) NOT NULL,
+    prev_state VARCHAR(255),
+    UNIQUE (event_id)
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS state_events_event_id ON state_events (event_id);
 CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
 CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
 CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    CONSTRAINT curr_uniq UNIQUE (room_id, type, state_key) ON CONFLICT REPLACE
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    state_key VARCHAR(255) NOT NULL,
+    UNIQUE (event_id),
+    UNIQUE (room_id, type, state_key)
 );
 
-CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
 CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
 CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
 CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id TEXT NOT NULL,
-    user_id TEXT NOT NULL,
-    sender TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    membership TEXT NOT NULL
+    event_id VARCHAR(255) NOT NULL,
+    user_id VARCHAR(255) NOT NULL,
+    sender VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    membership VARCHAR(255) NOT NULL,
+    UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
 CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
-    event_id TEXT NOT NULL,
-    feedback_type TEXT,
-    target_event_id TEXT,
-    sender TEXT,
-    room_id TEXT
+    event_id VARCHAR(255) NOT NULL,
+    feedback_type VARCHAR(255),
+    target_event_id VARCHAR(255),
+    sender VARCHAR(255),
+    room_id VARCHAR(255),
+    UNIQUE (event_id)
 );
 
 CREATE TABLE IF NOT EXISTS topics(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    topic TEXT NOT NULL
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    topic VARCHAR(255) NOT NULL,
+    UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
-    event_id TEXT NOT NULL,
-    room_id TEXT NOT NULL,
-    name TEXT NOT NULL
+    event_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    name VARCHAR(255) NOT NULL,
+    UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
 CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
-    room_id TEXT PRIMARY KEY NOT NULL,
+    room_id VARCHAR(255) PRIMARY KEY NOT NULL,
     is_public INTEGER,
-    creator TEXT
+    creator VARCHAR(255)
 );
 
 CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id TEXT NOT NULL,
-    host TEXT NOT NULL,
-    CONSTRAINT room_hosts_uniq UNIQUE (room_id, host) ON CONFLICT IGNORE
+    room_id VARCHAR(255) NOT NULL,
+    host VARCHAR(255) NOT NULL,
+    UNIQUE (room_id, host)
 );
 
 CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index a9e0a4fe0d..7a5afe0946 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -13,19 +13,19 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name TEXT, -- Server name.
-  fingerprint TEXT, -- Certificate fingerprint.
-  from_server TEXT, -- Which key server the certificate was fetched from.
+  server_name VARCHAR(255), -- Server name.
+  fingerprint VARCHAR(255), -- Certificate fingerprint.
+  from_server VARCHAR(255), -- Which key server the certificate was fetched from.
   ts_added_ms INTEGER, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
-  CONSTRAINT uniqueness UNIQUE (server_name, fingerprint)
+  UNIQUE (server_name, fingerprint)
 );
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name TEXT, -- Server name.
-  key_id TEXT, -- Key version.
-  from_server TEXT, -- Which key server the key was fetched form.
+  server_name VARCHAR(255), -- Server name.
+  key_id VARCHAR(255), -- Key version.
+  from_server VARCHAR(255), -- Which key server the key was fetched form.
   ts_added_ms INTEGER, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
-  CONSTRAINT uniqueness UNIQUE (server_name, key_id)
+  UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index afdf48cbfb..1e84c1d960 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -14,23 +14,23 @@
  */
 
 CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id TEXT, -- The id used to refer to the media.
-    media_type TEXT, -- The MIME-type of the media.
+    media_id VARCHAR(255), -- The id used to refer to the media.
+    media_type VARCHAR(255), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
     created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
-    user_id TEXT, -- The user who uploaded the file.
-    CONSTRAINT uniqueness UNIQUE (media_id)
+    upload_name VARCHAR(255), -- The name the media was uploaded with.
+    user_id VARCHAR(255), -- The user who uploaded the file.
+    UNIQUE (media_id)
 );
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id TEXT, -- The id used to refer to the media.
+    media_id VARCHAR(255), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
-    thumbnail_method TEXT, -- The method used to make the thumbnail.
+    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    CONSTRAINT uniqueness UNIQUE (
+    UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
     )
 );
@@ -39,29 +39,29 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media on that server.
-    media_type TEXT, -- The MIME-type of the media.
+    media_origin VARCHAR(255), -- The remote HS the media came from.
+    media_id VARCHAR(255), -- The id used to refer to the media on that server.
+    media_type VARCHAR(255), -- The MIME-type of the media.
     created_ts INTEGER, -- When the content was uploaded in ms.
-    upload_name TEXT, -- The name the media was uploaded with.
+    upload_name VARCHAR(255), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (media_origin, media_id)
+    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    UNIQUE (media_origin, media_id)
 );
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin TEXT, -- The remote HS the media came from.
-    media_id TEXT, -- The id used to refer to the media.
+    media_origin VARCHAR(255), -- The remote HS the media came from.
+    media_id VARCHAR(255), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method TEXT, -- The method used to make the thumbnail
-    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail
+    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id TEXT, -- The name used to store the media on disk.
-    CONSTRAINT uniqueness UNIQUE (
+    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    UNIQUE (
         media_origin, media_id, thumbnail_width, thumbnail_height,
-        thumbnail_type, thumbnail_type
-    )
+        thumbnail_type
+     )
 );
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index f9f8db9697..5198123499 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -13,26 +13,26 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS presence(
-  user_id INTEGER NOT NULL,
+  user_id VARCHAR(255) NOT NULL,
   state INTEGER,
-  status_msg TEXT,
+  status_msg VARCHAR(255),
   mtime INTEGER, -- miliseconds since last state change
-  FOREIGN KEY(user_id) REFERENCES users(id)
+  UNIQUE(user_id)
 );
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id INTEGER NOT NULL,
-  observer_user_id TEXT, -- a UserID,
-  FOREIGN KEY(observed_user_id) REFERENCES users(id)
+  observed_user_id VARCHAR(255) NOT NULL,
+  observer_user_id VARCHAR(255), -- a UserID,
+  UNIQUE(observed_user_id)
 );
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
-  user_id INTEGER NOT NULL,
-  observed_user_id TEXT, -- a UserID,
+  user_id VARCHAR(255) NOT NULL,
+  observed_user_id VARCHAR(255), -- a UserID,
   accepted BOOLEAN,
-  FOREIGN KEY(user_id) REFERENCES users(id)
+  UNIQUE(user_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index f06a528b4d..9b22c9b1b9 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS profiles(
-    user_id INTEGER NOT NULL,
-    displayname TEXT,
-    avatar_url TEXT,
-    FOREIGN KEY(user_id) REFERENCES users(id)
+    user_id VARCHAR(255) NOT NULL,
+    displayname VARCHAR(255),
+    avatar_url VARCHAR(255),
+    UNIQUE(user_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 5011d95db8..774584d7c5 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -13,9 +13,9 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS redactions (
-    event_id TEXT NOT NULL,
-    redacts TEXT NOT NULL,
-    CONSTRAINT ev_uniq UNIQUE (event_id)
+    event_id VARCHAR(255) NOT NULL,
+    redacts VARCHAR(255) NOT NULL,
+    UNIQUE (event_id)
 );
 
 CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 0d2df01603..dd51c31c1f 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -14,14 +14,12 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias TEXT NOT NULL,
-    room_id TEXT NOT NULL
+    room_alias VARCHAR(255) NOT NULL,
+    room_id VARCHAR(255) NOT NULL,
+    UNIQUE (room_alias)
 );
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias TEXT NOT NULL,
-    server TEXT NOT NULL
+    room_alias VARCHAR(255) NOT NULL,
+    server VARCHAR(255) NOT NULL
 );
-
-
-
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 1fe8f1e430..408a2d3914 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -15,33 +15,26 @@
 
 CREATE TABLE IF NOT EXISTS state_groups(
     id INTEGER PRIMARY KEY,
-    room_id TEXT NOT NULL,
-    event_id TEXT NOT NULL
+    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(255) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group INTEGER NOT NULL,
-    room_id TEXT NOT NULL,
-    type TEXT NOT NULL,
-    state_key TEXT NOT NULL,
-    event_id TEXT NOT NULL
+    room_id VARCHAR(255) NOT NULL,
+    type VARCHAR(255) NOT NULL,
+    state_key VARCHAR(255) NOT NULL,
+    event_id VARCHAR(255) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id TEXT NOT NULL,
+    event_id VARCHAR(255) NOT NULL,
     state_group INTEGER NOT NULL,
-    CONSTRAINT event_to_state_groups_uniq UNIQUE (event_id)
+    UNIQUE (event_id)
 );
 
 CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
 
-CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(
-    state_group
-);
-CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(
-    room_id, type, state_key
-);
-
-CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(
-    event_id
-);
\ No newline at end of file
+CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group);
+CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
+CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
\ No newline at end of file
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 2d30f99b06..0158308a38 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -14,34 +14,31 @@
  */
 -- Stores what transaction ids we have received and what our response was
 CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id TEXT, 
-    origin TEXT, 
+    transaction_id VARCHAR(255),
+    origin VARCHAR(255),
     ts INTEGER,
     response_code INTEGER,
-    response_json TEXT,
+    response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
-    CONSTRAINT uniquesss UNIQUE (transaction_id, origin) ON CONFLICT REPLACE
+    UNIQUE (transaction_id, origin)
 );
 
-CREATE UNIQUE INDEX IF NOT EXISTS transactions_txid ON received_transactions(transaction_id, origin);
 CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
 
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
-    transaction_id TEXT,
-    destination TEXT,
+    id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering
+    transaction_id VARCHAR(255),
+    destination VARCHAR(255),
     response_code INTEGER DEFAULT 0,
-    response_json TEXT,
+    response_json BLOB,
     ts INTEGER
 );
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(
-    destination
-);
+CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
 -- So that we can do an efficient look up of all transactions that have yet to be successfully
 -- sent.
@@ -51,18 +48,17 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c
 -- For sent transactions only.
 CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
-    destination TEXT,
-    pdu_id TEXT,
-    pdu_origin TEXT
+    destination VARCHAR(255),
+    pdu_id VARCHAR(255),
+    pdu_origin VARCHAR(255),
+    UNIQUE (transaction_id, destination)
 );
 
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_index ON transaction_id_to_pdu(transaction_id, destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
-    destination TEXT PRIMARY KEY,
+    destination VARCHAR(255) PRIMARY KEY,
     retry_last_ts INTEGER,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 08ccfdac0a..49a122eed6 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,33 +13,32 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT,
-    password_hash TEXT,
+    id INTEGER PRIMARY KEY,
+    name VARCHAR(255),
+    password_hash VARBINARY(255),
     creation_ts INTEGER,
     admin BOOL DEFAULT 0 NOT NULL,
-    UNIQUE(name) ON CONFLICT ROLLBACK
+    UNIQUE(name)
 );
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY,
     user_id INTEGER NOT NULL,
-    device_id TEXT,
-    token TEXT NOT NULL,
+    device_id VARCHAR(255),
+    token VARCHAR(255) NOT NULL,
     last_used INTEGER,
     FOREIGN KEY(user_id) REFERENCES users(id),
-    UNIQUE(token) ON CONFLICT ROLLBACK
+    UNIQUE(token)
 );
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user TEXT NOT NULL,
-    access_token TEXT NOT NULL,
-    device_id TEXT,
-    ip TEXT NOT NULL,
-    user_agent TEXT NOT NULL,
+    user VARCHAR(255) NOT NULL,
+    access_token VARCHAR(255) NOT NULL,
+    device_id VARCHAR(255),
+    ip VARCHAR(255) NOT NULL,
+    user_agent VARCHAR(255) NOT NULL,
     last_seen INTEGER NOT NULL,
-    CONSTRAINT user_ip UNIQUE (user, access_token, ip, user_agent) ON CONFLICT REPLACE
+    UNIQUE (user, access_token, ip, user_agent)
 );
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index 0431e2d051..28762861e5 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -14,17 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS schema_version(
-    Lock char(1) NOT NULL DEFAULT 'X',  -- Makes sure this table only has one row.
-    version INTEGER NOT NULL,
-    upgraded BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
-    CONSTRAINT schema_version_lock_x CHECK (Lock='X')
-    CONSTRAINT schema_version_lock_uniq UNIQUE (Lock)
+    `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE,  -- Makes sure this table only has one row.
+    `version` INTEGER NOT NULL,
+    `upgraded` BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
+    CHECK (`Lock`='X')
 );
 
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
-    version INTEGER NOT NULL,
-    file TEXT NOT NULL,
-    CONSTRAINT schema_deltas_ver_file UNIQUE (version, file) ON CONFLICT IGNORE
+    `version` INTEGER NOT NULL,
+    `file` VARCHAR(255) NOT NULL,
+    UNIQUE(version, file)
 );
 
 CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version);
-- 
cgit 1.5.1


From fae059cc18c0aab25b79d5352aa98ccfda8c637a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 19 Mar 2015 13:42:39 +0000
Subject: Fix up schemas to work with mariadb

---
 synapse/storage/schema/delta/12/v12.sql            | 16 ++++++-------
 synapse/storage/schema/delta/13/v13.sql            | 10 ++++----
 synapse/storage/schema/delta/14/v14.sql            |  4 ++--
 .../storage/schema/full_schemas/11/event_edges.sql | 18 +++++++-------
 .../schema/full_schemas/11/event_signatures.sql    |  8 +++----
 synapse/storage/schema/full_schemas/11/im.sql      | 28 +++++++++++-----------
 synapse/storage/schema/full_schemas/11/keys.sql    |  8 +++----
 .../schema/full_schemas/11/media_repository.sql    | 12 +++++-----
 .../storage/schema/full_schemas/11/presence.sql    | 10 ++++----
 .../storage/schema/full_schemas/11/profiles.sql    |  2 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  2 +-
 .../schema/full_schemas/11/room_aliases.sql        |  4 ++--
 synapse/storage/schema/full_schemas/11/state.sql   | 12 +++++-----
 .../schema/full_schemas/11/transactions.sql        | 14 +++++------
 synapse/storage/schema/full_schemas/11/users.sql   | 18 +++++++-------
 15 files changed, 82 insertions(+), 84 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index fbe8486b96..42ea1b2ff5 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -18,11 +18,11 @@ CREATE TABLE IF NOT EXISTS rejections(
     reason VARCHAR(255) NOT NULL,
     last_check VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id INTEGER PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(255) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
@@ -37,10 +37,10 @@ CREATE TABLE IF NOT EXISTS pushers (
   last_success BIGINT,
   failing_since BIGINT,
   UNIQUE (app_id, pushkey)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id INTEGER PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(255) NOT NULL,
   rule_id VARCHAR(255) NOT NULL,
   priority_class TINYINT NOT NULL,
@@ -48,16 +48,16 @@ CREATE TABLE IF NOT EXISTS push_rules (
   conditions VARCHAR(255) NOT NULL,
   actions VARCHAR(255) NOT NULL,
   UNIQUE(user_name, rule_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(255),
-  filter_id INTEGER,
+  filter_id BIGINT,
   filter_json BLOB
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
     user_id, filter_id
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 68d4ebd896..f0a5daf445 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -14,18 +14,18 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id INTEGER PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
     url VARCHAR(255),
     token VARCHAR(255),
     hs_token VARCHAR(255),
     sender VARCHAR(255),
     UNIQUE(token)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id INTEGER PRIMARY KEY,
-    as_id INTEGER NOT NULL,
+    id BIGINT PRIMARY KEY,
+    as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(255),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index 2b50adbd61..a1260c5c1f 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,9 +1,9 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id INTEGER PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(255) NOT NULL,
   rule_id VARCHAR(255) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 823653f2e9..910e25dfce 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges(
     room_id VARCHAR(255) NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
 CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
@@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth(
     room_id VARCHAR(255) NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
@@ -57,9 +57,9 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(255) NOT NULL,
     destination VARCHAR(255) NOT NULL,
-    delivered_ts INTEGER DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
 
@@ -70,11 +70,11 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     type VARCHAR(255) NOT NULL,
     state_key VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
-);
+) ENGINE = INNODB;
 CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
 
 
@@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth(
     auth_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
 CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index a7c3acf775..334d7c8680 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes (
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
 
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes (
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
 
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     key_id VARCHAR(255),
     signature BLOB,
     UNIQUE (event_id, signature_name, key_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
@@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes(
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 8feacdbe08..d58106def8 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering INTEGER PRIMARY KEY,
-    topological_ordering INTEGER NOT NULL,
+    stream_ordering BIGINT PRIMARY KEY,
+    topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(255) NOT NULL,
     type VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
@@ -23,9 +23,9 @@ CREATE TABLE IF NOT EXISTS events(
     unrecognized_keys BLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
-    depth INTEGER DEFAULT 0 NOT NULL,
+    depth BIGINT DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
 CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
@@ -39,19 +39,19 @@ CREATE TABLE IF NOT EXISTS event_json(
     internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
 
-CREATE TABLE state_events(
+CREATE TABLE IF NOT EXISTS state_events(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     type VARCHAR(255) NOT NULL,
     state_key VARCHAR(255) NOT NULL,
     prev_state VARCHAR(255),
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
 CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
@@ -65,7 +65,7 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     state_key VARCHAR(255) NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
 CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
@@ -78,7 +78,7 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     room_id VARCHAR(255) NOT NULL,
     membership VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
@@ -90,14 +90,14 @@ CREATE TABLE IF NOT EXISTS feedback(
     sender VARCHAR(255),
     room_id VARCHAR(255),
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     topic VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
@@ -112,14 +112,14 @@ CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
     room_id VARCHAR(255) PRIMARY KEY NOT NULL,
-    is_public INTEGER,
+    is_public BOOL,
     creator VARCHAR(255)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS room_hosts(
     room_id VARCHAR(255) NOT NULL,
     host VARCHAR(255) NOT NULL,
     UNIQUE (room_id, host)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 7a5afe0946..c0f2ec29bb 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -16,16 +16,16 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name VARCHAR(255), -- Server name.
   fingerprint VARCHAR(255), -- Certificate fingerprint.
   from_server VARCHAR(255), -- Which key server the certificate was fetched from.
-  ts_added_ms INTEGER, -- When the certifcate was added.
+  ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(255), -- Server name.
   key_id VARCHAR(255), -- Key version.
   from_server VARCHAR(255), -- Which key server the key was fetched form.
-  ts_added_ms INTEGER, -- When the key was added.
+  ts_added_ms BIGINT, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 1e84c1d960..8bc84dc24d 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -17,11 +17,11 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     media_id VARCHAR(255), -- The id used to refer to the media.
     media_type VARCHAR(255), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
-    created_ts INTEGER, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(255), -- The name the media was uploaded with.
     user_id VARCHAR(255), -- The user who uploaded the file.
     UNIQUE (media_id)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     media_id VARCHAR(255), -- The id used to refer to the media.
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
     )
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
@@ -42,12 +42,12 @@ CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_origin VARCHAR(255), -- The remote HS the media came from.
     media_id VARCHAR(255), -- The id used to refer to the media on that server.
     media_type VARCHAR(255), -- The MIME-type of the media.
-    created_ts INTEGER, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(255), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(255), -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
     media_origin VARCHAR(255), -- The remote HS the media came from.
@@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
      )
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index 5198123499..8031321083 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -14,11 +14,11 @@
  */
 CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(255) NOT NULL,
-  state INTEGER,
+  state VARCHAR(20),
   status_msg VARCHAR(255),
-  mtime INTEGER, -- miliseconds since last state change
+  mtime BIGINT, -- miliseconds since last state change
   UNIQUE(user_id)
-);
+) ENGINE = INNODB;
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound(
   observed_user_id VARCHAR(255) NOT NULL,
   observer_user_id VARCHAR(255), -- a UserID,
   UNIQUE(observed_user_id)
-);
+) ENGINE = INNODB;
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
@@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list(
   observed_user_id VARCHAR(255), -- a UserID,
   accepted BOOLEAN,
   UNIQUE(user_id)
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 9b22c9b1b9..32defe2f79 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles(
     displayname VARCHAR(255),
     avatar_url VARCHAR(255),
     UNIQUE(user_id)
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 774584d7c5..ba93e860f6 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions (
     event_id VARCHAR(255) NOT NULL,
     redacts VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
 CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index dd51c31c1f..1e706aac2b 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases(
     room_alias VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (room_alias)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
     room_alias VARCHAR(255) NOT NULL,
     server VARCHAR(255) NOT NULL
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 408a2d3914..be9dc2920d 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -14,24 +14,24 @@
  */
 
 CREATE TABLE IF NOT EXISTS state_groups(
-    id INTEGER PRIMARY KEY,
+    id VARCHAR(20) PRIMARY KEY,
     room_id VARCHAR(255) NOT NULL,
     event_id VARCHAR(255) NOT NULL
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
-    state_group INTEGER NOT NULL,
+    state_group VARCHAR(20) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     type VARCHAR(255) NOT NULL,
     state_key VARCHAR(255) NOT NULL,
     event_id VARCHAR(255) NOT NULL
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
     event_id VARCHAR(255) NOT NULL,
-    state_group INTEGER NOT NULL,
+    state_group VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 0158308a38..cde6919e42 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -16,12 +16,12 @@
 CREATE TABLE IF NOT EXISTS received_transactions(
     transaction_id VARCHAR(255),
     origin VARCHAR(255),
-    ts INTEGER,
+    ts BIGINT,
     response_code INTEGER,
     response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
@@ -29,13 +29,13 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id INTEGER PRIMARY KEY, -- This is used to apply insertion ordering
+    id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
     transaction_id VARCHAR(255),
     destination VARCHAR(255),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
     ts INTEGER
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination);
@@ -52,13 +52,13 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     pdu_id VARCHAR(255),
     pdu_origin VARCHAR(255),
     UNIQUE (transaction_id, destination)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(255) PRIMARY KEY,
-    retry_last_ts INTEGER,
+    retry_last_ts BIGINT,
     retry_interval INTEGER
-);
+) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 49a122eed6..55bffb22f3 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,23 +13,21 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    id INTEGER PRIMARY KEY,
     name VARCHAR(255),
     password_hash VARBINARY(255),
-    creation_ts INTEGER,
+    creation_ts BIGINT,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id INTEGER PRIMARY KEY,
-    user_id INTEGER NOT NULL,
+    id INTEGER PRIMARY KEY AUTO_INCREMENT,
+    user_id VARCHAR(255) NOT NULL,
     device_id VARCHAR(255),
     token VARCHAR(255) NOT NULL,
-    last_used INTEGER,
-    FOREIGN KEY(user_id) REFERENCES users(id),
+    last_used BIGINT,
     UNIQUE(token)
-);
+) ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS user_ips (
     user VARCHAR(255) NOT NULL,
@@ -37,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(255),
     ip VARCHAR(255) NOT NULL,
     user_agent VARCHAR(255) NOT NULL,
-    last_seen INTEGER NOT NULL,
+    last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
-);
+) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-- 
cgit 1.5.1


From 58ed393235381b37182212547af7d65e2f6fa384 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 19 Mar 2015 15:12:05 +0000
Subject: Remove redundant key

---
 synapse/storage/schema/full_schemas/11/im.sql | 1 -
 1 file changed, 1 deletion(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index d58106def8..9849e969be 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -27,7 +27,6 @@ CREATE TABLE IF NOT EXISTS events(
     UNIQUE (event_id)
 ) ENGINE = INNODB;
 
-CREATE INDEX IF NOT EXISTS events_event_id ON events (event_id);
 CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
 CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
 CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
-- 
cgit 1.5.1


From 0d0610870ddd3c47fe397f1d026bf1ca01dbdf74 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 20 Mar 2015 10:55:31 +0000
Subject: Fix up schemas some more

---
 synapse/storage/schema/delta/12/v12.sql                 | 2 +-
 synapse/storage/schema/full_schemas/11/event_edges.sql  | 2 +-
 synapse/storage/schema/full_schemas/11/transactions.sql | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 42ea1b2ff5..b526109e6e 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS user_filters(
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
     user_id, filter_id
-) ENGINE = INNODB;
+);
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 910e25dfce..0f53488e92 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
 
 CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
-) ENGINE = INNODB;
+);
 CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
 
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index cde6919e42..0570bf95d9 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     destination VARCHAR(255),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
-    ts INTEGER
+    ts BIGINT
 ) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-- 
cgit 1.5.1


From c7023f21555a0adf0d8bb5040c817a8198bbf5a8 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Tue, 24 Mar 2015 17:24:15 +0000
Subject: 1) Pushers are now associated with an access token 2) Change places
 where we mean unauthenticated to 401, not 403, in C/S v2: hack so it stays as
 403 in v1 because web client relies on it.

---
 synapse/api/auth.py                     | 43 +++++++++++++++++++--------------
 synapse/push/pusherpool.py              |  9 ++++---
 synapse/rest/client/v1/base.py          |  2 +-
 synapse/rest/client/v1/pusher.py        |  3 ++-
 synapse/server.py                       | 10 ++++++++
 synapse/storage/pusher.py               |  3 ++-
 synapse/storage/registration.py         |  2 +-
 synapse/storage/schema/delta/15/v15.sql |  2 ++
 8 files changed, 48 insertions(+), 26 deletions(-)
 create mode 100644 synapse/storage/schema/delta/15/v15.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/api/auth.py b/synapse/api/auth.py
index 64f605b962..d08faf23f1 100644
--- a/synapse/api/auth.py
+++ b/synapse/api/auth.py
@@ -40,6 +40,7 @@ class Auth(object):
         self.hs = hs
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
+        self.TOKEN_NOT_FOUND_HTTP_STATUS = 401
 
     def check(self, event, auth_events):
         """ Checks if this event is correctly authed.
@@ -373,7 +374,9 @@ class Auth(object):
 
             defer.returnValue((user, ClientInfo(device_id, token_id)))
         except KeyError:
-            raise AuthError(403, "Missing access token.")
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token."
+            )
 
     @defer.inlineCallbacks
     def get_user_by_token(self, token):
@@ -387,21 +390,20 @@ class Auth(object):
         Raises:
             AuthError if no user by that token exists or the token is invalid.
         """
-        try:
-            ret = yield self.store.get_user_by_token(token)
-            if not ret:
-                raise StoreError(400, "Unknown token")
-            user_info = {
-                "admin": bool(ret.get("admin", False)),
-                "device_id": ret.get("device_id"),
-                "user": UserID.from_string(ret.get("name")),
-                "token_id": ret.get("token_id", None),
-            }
+        ret = yield self.store.get_user_by_token(token)
+        if not ret:
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Unrecognised access token.",
+                errcode=Codes.UNKNOWN_TOKEN
+            )
+        user_info = {
+            "admin": bool(ret.get("admin", False)),
+            "device_id": ret.get("device_id"),
+            "user": UserID.from_string(ret.get("name")),
+            "token_id": ret.get("token_id", None),
+        }
 
-            defer.returnValue(user_info)
-        except StoreError:
-            raise AuthError(403, "Unrecognised access token.",
-                            errcode=Codes.UNKNOWN_TOKEN)
+        defer.returnValue(user_info)
 
     @defer.inlineCallbacks
     def get_appservice_by_req(self, request):
@@ -409,11 +411,16 @@ class Auth(object):
             token = request.args["access_token"][0]
             service = yield self.store.get_app_service_by_token(token)
             if not service:
-                raise AuthError(403, "Unrecognised access token.",
-                                errcode=Codes.UNKNOWN_TOKEN)
+                raise AuthError(
+                    self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                    "Unrecognised access token.",
+                    errcode=Codes.UNKNOWN_TOKEN
+                )
             defer.returnValue(service)
         except KeyError:
-            raise AuthError(403, "Missing access token.")
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token."
+            )
 
     def is_server_admin(self, user):
         return self.store.is_server_admin(user)
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index 90babd7224..f75eebf8bf 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -57,7 +57,7 @@ class PusherPool:
         self._start_pushers(pushers)
 
     @defer.inlineCallbacks
-    def add_pusher(self, user_name, profile_tag, kind, app_id,
+    def add_pusher(self, user_name, access_token, profile_tag, kind, app_id,
                    app_display_name, device_display_name, pushkey, lang, data):
         # we try to create the pusher just to validate the config: it
         # will then get pulled out of the database,
@@ -79,17 +79,18 @@ class PusherPool:
             "failing_since": None
         })
         yield self._add_pusher_to_store(
-            user_name, profile_tag, kind, app_id,
+            user_name, access_token, profile_tag, kind, app_id,
             app_display_name, device_display_name,
             pushkey, lang, data
         )
 
     @defer.inlineCallbacks
-    def _add_pusher_to_store(self, user_name, profile_tag, kind, app_id,
-                             app_display_name, device_display_name,
+    def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind,
+                             app_id, app_display_name, device_display_name,
                              pushkey, lang, data):
         yield self.store.add_pusher(
             user_name=user_name,
+            access_token=access_token,
             profile_tag=profile_tag,
             kind=kind,
             app_id=app_id,
diff --git a/synapse/rest/client/v1/base.py b/synapse/rest/client/v1/base.py
index 72332bdb10..504a5e432f 100644
--- a/synapse/rest/client/v1/base.py
+++ b/synapse/rest/client/v1/base.py
@@ -48,5 +48,5 @@ class ClientV1RestServlet(RestServlet):
         self.hs = hs
         self.handlers = hs.get_handlers()
         self.builder_factory = hs.get_event_builder_factory()
-        self.auth = hs.get_auth()
+        self.auth = hs.get_v1auth()
         self.txns = HttpTransactionStore()
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 6045e86f34..87e89c9305 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -27,7 +27,7 @@ class PusherRestServlet(ClientV1RestServlet):
 
     @defer.inlineCallbacks
     def on_POST(self, request):
-        user, _ = yield self.auth.get_user_by_req(request)
+        user, client = yield self.auth.get_user_by_req(request)
 
         content = _parse_json(request)
 
@@ -54,6 +54,7 @@ class PusherRestServlet(ClientV1RestServlet):
         try:
             yield pusher_pool.add_pusher(
                 user_name=user.to_string(),
+                access_token=client.token_id,
                 profile_tag=content['profile_tag'],
                 kind=content['kind'],
                 app_id=content['app_id'],
diff --git a/synapse/server.py b/synapse/server.py
index c7772244ba..4c4f6ca239 100644
--- a/synapse/server.py
+++ b/synapse/server.py
@@ -65,6 +65,7 @@ class BaseHomeServer(object):
         'replication_layer',
         'datastore',
         'handlers',
+        'v1auth',
         'auth',
         'rest_servlet_factory',
         'state_handler',
@@ -182,6 +183,15 @@ class HomeServer(BaseHomeServer):
     def build_auth(self):
         return Auth(self)
 
+    def build_v1auth(self):
+        orf = Auth(self)
+        # Matrix spec makes no reference to what HTTP status code is returned,
+        # but the V1 API uses 403 where it means 401, and the webclient
+        # relies on this behaviour, so V1 gets its own copy of the auth
+        # with backwards compat behaviour.
+        orf.TOKEN_NOT_FOUND_HTTP_STATUS = 403
+        return orf
+
     def build_state_handler(self):
         return StateHandler(self)
 
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 000502b4ff..1ef8e06ac6 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -95,7 +95,7 @@ class PusherStore(SQLBaseStore):
         defer.returnValue(ret)
 
     @defer.inlineCallbacks
-    def add_pusher(self, user_name, profile_tag, kind, app_id,
+    def add_pusher(self, user_name, access_token, profile_tag, kind, app_id,
                    app_display_name, device_display_name,
                    pushkey, pushkey_ts, lang, data):
         try:
@@ -107,6 +107,7 @@ class PusherStore(SQLBaseStore):
                 ),
                 dict(
                     user_name=user_name,
+                    access_token=access_token,
                     kind=kind,
                     profile_tag=profile_tag,
                     app_display_name=app_display_name,
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 0364d10858..f61d8fdb6a 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -174,4 +174,4 @@ class RegistrationStore(SQLBaseStore):
         if rows:
             return rows[0]
 
-        raise StoreError(404, "Token not found.")
+        return None
diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql
new file mode 100644
index 0000000000..fc3e436877
--- /dev/null
+++ b/synapse/storage/schema/delta/15/v15.sql
@@ -0,0 +1,2 @@
+ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL;
+
-- 
cgit 1.5.1


From 0e8f5095c7e7075b249ad53a9f60a4d2fdeeaaed Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 25 Mar 2015 17:15:20 +0000
Subject: Fix unicode database support

---
 synapse/app/homeserver.py                          | 47 +++++++++++++---------
 synapse/handlers/login.py                          |  2 +-
 synapse/rest/client/v1/profile.py                  |  7 +++-
 synapse/storage/__init__.py                        | 12 ++++--
 synapse/storage/_base.py                           |  4 ++
 synapse/storage/events.py                          |  8 ++--
 synapse/storage/keys.py                            |  4 +-
 synapse/storage/profile.py                         | 12 +++++-
 synapse/storage/registration.py                    | 18 +++++++--
 synapse/storage/room.py                            |  1 +
 .../schema/full_schemas/11/media_repository.sql    |  2 +-
 .../storage/schema/full_schemas/11/profiles.sql    |  2 +-
 .../schema/full_schemas/11/transactions.sql        |  1 -
 synapse/storage/signatures.py                      | 10 ++---
 synapse/storage/transactions.py                    |  2 +-
 15 files changed, 88 insertions(+), 44 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 394e93e6c2..beab6ffc7a 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -110,14 +110,12 @@ class SynapseHomeServer(HomeServer):
             return None
 
     def build_db_pool(self):
-        name = self.db_config.pop("name", None)
-        if name == "MySQLdb":
-            return adbapi.ConnectionPool(
-                name,
-                **self.db_config
-            )
+        name = self.db_config["name"]
 
-        raise RuntimeError("Unsupported database type")
+        return adbapi.ConnectionPool(
+            name,
+            **self.db_config.get("args", {})
+        )
 
     def create_resource_tree(self, redirect_root_to_web_client):
         """Create the resource tree for this Home Server.
@@ -323,7 +321,7 @@ def change_resource_limit(soft_file_no):
         resource.setrlimit(resource.RLIMIT_NOFILE, (soft_file_no, hard))
 
         logger.info("Set file limit to: %d", soft_file_no)
-    except (ValueError, resource.error) as e:
+    except (   ValueError, resource.error) as e:
         logger.warn("Failed to set file limit: %s", e)
 
 
@@ -363,20 +361,33 @@ def setup(config_options):
     if config.database_config:
         with open(config.database_config, 'r') as f:
             db_config = yaml.safe_load(f)
-
-        name = db_config.get("name", None)
-        if name == "MySQLdb":
-            db_config.update({
-                "sql_mode": "TRADITIONAL",
-                "charset": "utf8",
-                "use_unicode": True,
-            })
     else:
         db_config = {
             "name": "sqlite3",
             "database": config.database_path,
         }
 
+    db_config = {
+        k: v for k, v in db_config.items()
+        if not k.startswith("cp_")
+    }
+
+    name = db_config.get("name", None)
+    if name in ["MySQLdb", "mysql.connector"]:
+        db_config.setdefault("args", {}).update({
+            "sql_mode": "TRADITIONAL",
+            "charset": "utf8",
+            "use_unicode": True,
+        })
+    elif name == "sqlite3":
+        db_config.setdefault("args", {}).update({
+            "cp_min": 1,
+            "cp_max": 1,
+            "cp_openfun": prepare_database,
+        })
+    else:
+        raise RuntimeError("Unsupported database type '%s'" % (name,))
+
     hs = SynapseHomeServer(
         config.server_name,
         domain_with_port=domain_with_port,
@@ -401,8 +412,8 @@ def setup(config_options):
         # with sqlite3.connect(db_name) as db_conn:
         #     prepare_sqlite3_database(db_conn)
         #     prepare_database(db_conn)
-        import MySQLdb
-        db_conn = MySQLdb.connect(**db_config)
+        import mysql.connector
+        db_conn = mysql.connector.connect(**db_config.get("args", {}))
         prepare_database(db_conn)
     except UpgradeDatabaseException:
         sys.stderr.write(
diff --git a/synapse/handlers/login.py b/synapse/handlers/login.py
index 7447800460..76647c7941 100644
--- a/synapse/handlers/login.py
+++ b/synapse/handlers/login.py
@@ -57,7 +57,7 @@ class LoginHandler(BaseHandler):
             logger.warn("Attempted to login as %s but they do not exist", user)
             raise LoginError(403, "", errcode=Codes.FORBIDDEN)
 
-        stored_hash = user_info[0]["password_hash"]
+        stored_hash = user_info["password_hash"]
         if bcrypt.checkpw(password, stored_hash):
             # generate an access token and store it.
             token = self.reg_handler._generate_token(user)
diff --git a/synapse/rest/client/v1/profile.py b/synapse/rest/client/v1/profile.py
index 1e77eb49cf..7387b4adb9 100644
--- a/synapse/rest/client/v1/profile.py
+++ b/synapse/rest/client/v1/profile.py
@@ -19,9 +19,13 @@ from twisted.internet import defer
 from .base import ClientV1RestServlet, client_path_pattern
 from synapse.types import UserID
 
+import logging
 import simplejson as json
 
 
+logger = logging.getLogger(__name__)
+
+
 class ProfileDisplaynameRestServlet(ClientV1RestServlet):
     PATTERN = client_path_pattern("/profile/(?P<user_id>[^/]*)/displayname")
 
@@ -47,7 +51,8 @@ class ProfileDisplaynameRestServlet(ClientV1RestServlet):
             defer.returnValue((400, "Unable to parse name"))
 
         yield self.handlers.profile_handler.set_displayname(
-            user, auth_user, new_name)
+            user, auth_user, new_name
+        )
 
         defer.returnValue((200, {}))
 
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index e03d55b00d..abde7d0df5 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -410,10 +410,14 @@ def executescript(txn, schema_path):
 
 
 def _get_or_create_schema_state(txn):
-    schema_path = os.path.join(
-        dir_path, "schema", "schema_version.sql",
-    )
-    executescript(txn, schema_path)
+    try:
+        # Bluntly try creating the schema_version tables.
+        schema_path = os.path.join(
+            dir_path, "schema", "schema_version.sql",
+        )
+        executescript(txn, schema_path)
+    except:
+        pass
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 1ea39bc0ad..76ec3ee93f 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -755,6 +755,8 @@ class SQLBaseStore(object):
             return None
 
         internal_metadata, js, redacted, rejected_reason = res
+        js = js.decode("utf8")
+        internal_metadata = internal_metadata.decode("utf8")
 
         start_time = update_counter("select_event", start_time)
 
@@ -779,9 +781,11 @@ class SQLBaseStore(object):
             sql_getevents_timer.inc_by(curr_time - last_time, desc)
             return curr_time
 
+        logger.debug("Got js: %r", js)
         d = json.loads(js)
         start_time = update_counter("decode_json", start_time)
 
+        logger.debug("Got internal_metadata: %r", internal_metadata)
         internal_metadata = json.loads(internal_metadata)
         start_time = update_counter("decode_internal", start_time)
 
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 4d636d3f46..69f598967e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -294,15 +294,17 @@ class EventsStore(SQLBaseStore):
             )
 
             if is_new_state and not context.rejected:
-                self._simple_insert_txn(
+                self._simple_upsert_txn(
                     txn,
                     "current_state_events",
-                    {
-                        "event_id": event.event_id,
+                    keyvalues={
                         "room_id": event.room_id,
                         "type": event.type,
                         "state_key": event.state_key,
                     },
+                    values={
+                        "event_id": event.event_id,
+                    }
                 )
 
             for e_id, h in event.prev_state:
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 25fef79434..e6975a945b 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore):
                 "fingerprint": fingerprint,
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "tls_certificate": buffer(tls_certificate_bytes),
+                "tls_certificate": tls_certificate_bytes,
             },
         )
 
@@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore):
                 "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "verify_key": buffer(verify_key.encode()),
+                "verify_key": verify_key.encode(),
             },
         )
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index a6e52cb248..09778045bf 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from twisted.internet import defer
+
 from ._base import SQLBaseStore
 
 
@@ -24,19 +26,25 @@ class ProfileStore(SQLBaseStore):
             desc="create_profile",
         )
 
+    @defer.inlineCallbacks
     def get_profile_displayname(self, user_localpart):
-        return self._simple_select_one_onecol(
+        name = yield self._simple_select_one_onecol(
             table="profiles",
             keyvalues={"user_id": user_localpart},
             retcol="displayname",
             desc="get_profile_displayname",
         )
 
+        if name:
+            name = name.decode("utf8")
+
+        defer.returnValue(name)
+
     def set_profile_displayname(self, user_localpart, new_displayname):
         return self._simple_update_one(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname},
+            updatevalues={"displayname": new_displayname.encode("utf8")},
             desc="set_profile_displayname",
         )
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index fe26d6d62f..7258f7b2a5 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -81,13 +81,23 @@ class RegistrationStore(SQLBaseStore):
         txn.execute("INSERT INTO access_tokens(user_id, token) " +
                     "VALUES (?,?)", [user_id, token])
 
+    @defer.inlineCallbacks
     def get_user_by_id(self, user_id):
-        query = ("SELECT users.name, users.password_hash FROM users"
-                 " WHERE users.name = ?")
-        return self._execute(
-            "get_user_by_id", self.cursor_to_dict, query, user_id
+        user_info = yield self._simple_select_one(
+            table="users",
+            keyvalues={
+                "name": user_id,
+            },
+            retcols=["name", "password_hash"],
+            allow_none=True,
         )
 
+        if user_info:
+            user_info["password_hash"] = user_info["password_hash"].decode("utf8")
+
+        defer.returnValue(user_info)
+
+
     @cached()
     # TODO(paul): Currently there's no code to invalidate this cache. That
     #   means if/when we ever add internal ways to invalidate access tokens or
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index 501e947ad7..a1a76280fe 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -72,6 +72,7 @@ class RoomStore(SQLBaseStore):
             keyvalues={"room_id": room_id},
             retcols=RoomsTable.fields,
             desc="get_room",
+            allow_none=True,
         )
 
     @defer.inlineCallbacks
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 8bc84dc24d..d9559f5902 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -65,4 +65,4 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
 ) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
-    ON local_media_repository_thumbnails (media_id);
+    ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 32defe2f79..552645c56f 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -14,7 +14,7 @@
  */
 CREATE TABLE IF NOT EXISTS profiles(
     user_id VARCHAR(255) NOT NULL,
-    displayname VARCHAR(255),
+    displayname VARBINARY(255),
     avatar_url VARCHAR(255),
     UNIQUE(user_id)
 ) ENGINE = INNODB;
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 0570bf95d9..bd13bba8c2 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -38,7 +38,6 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
 ) ENGINE = INNODB;
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_dest_referenced ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
 -- So that we can do an efficient look up of all transactions that have yet to be successfully
 -- sent.
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 13ce335101..35bba854f9 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
 
@@ -99,7 +99,7 @@ class SignatureStore(SQLBaseStore):
             " WHERE event_id = ?"
         )
         txn.execute(query, (event_id, ))
-        return dict(txn.fetchall())
+        return {k: v for k, v in txn.fetchall()}
 
     def _store_event_reference_hash_txn(self, txn, event_id, algorithm,
                                         hash_bytes):
@@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
 
@@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "signature_name": signature_name,
                 "key_id": key_id,
-                "signature": buffer(signature_bytes),
+                "signature": signature_bytes,
             },
         )
 
@@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "prev_event_id": prev_event_id,
                 "algorithm": algorithm,
-                "hash": buffer(hash_bytes),
+                "hash": hash_bytes,
             },
         )
diff --git a/synapse/storage/transactions.py b/synapse/storage/transactions.py
index 03e1e3b808..e3e484fb2d 100644
--- a/synapse/storage/transactions.py
+++ b/synapse/storage/transactions.py
@@ -282,7 +282,7 @@ class TransactionStore(SQLBaseStore):
         query = (
             "UPDATE destinations"
             " SET retry_last_ts = ?, retry_interval = ?"
-            " WHERE destinations = ?"
+            " WHERE destination = ?"
         )
 
         txn.execute(
-- 
cgit 1.5.1


From c1a256cc4c82ce746eae8e719b9aa2344fe66177 Mon Sep 17 00:00:00 2001
From: David Baker <dave@matrix.org>
Date: Wed, 25 Mar 2015 19:06:22 +0000
Subject: Allow multiple pushers for a single app ID & pushkey, honouring the
 'append' flag in the API.

---
 synapse/push/__init__.py                | 10 +++++--
 synapse/push/pusherpool.py              | 47 ++++++++++++++++++++++++++-------
 synapse/rest/client/v1/pusher.py        | 13 ++++++++-
 synapse/storage/pusher.py               | 27 ++++++++++---------
 synapse/storage/schema/delta/15/v15.sql | 27 +++++++++++++++++--
 5 files changed, 96 insertions(+), 28 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/push/__init__.py b/synapse/push/__init__.py
index 0727f772a5..5575c847f9 100644
--- a/synapse/push/__init__.py
+++ b/synapse/push/__init__.py
@@ -253,7 +253,8 @@ class Pusher(object):
                 self.user_name, config, timeout=0)
             self.last_token = chunk['end']
             self.store.update_pusher_last_token(
-                self.app_id, self.pushkey, self.last_token)
+                self.app_id, self.pushkey, self.user_name, self.last_token
+            )
             logger.info("Pusher %s for user %s starting from token %s",
                         self.pushkey, self.user_name, self.last_token)
 
@@ -314,7 +315,7 @@ class Pusher(object):
                                 pk
                             )
                             yield self.hs.get_pusherpool().remove_pusher(
-                                self.app_id, pk
+                                self.app_id, pk, self.user_name
                             )
 
             if not self.alive:
@@ -326,6 +327,7 @@ class Pusher(object):
                 self.store.update_pusher_last_token_and_success(
                     self.app_id,
                     self.pushkey,
+                    self.user_name,
                     self.last_token,
                     self.clock.time_msec()
                 )
@@ -334,6 +336,7 @@ class Pusher(object):
                     self.store.update_pusher_failing_since(
                         self.app_id,
                         self.pushkey,
+                        self.user_name,
                         self.failing_since)
             else:
                 if not self.failing_since:
@@ -341,6 +344,7 @@ class Pusher(object):
                     self.store.update_pusher_failing_since(
                         self.app_id,
                         self.pushkey,
+                        self.user_name,
                         self.failing_since
                     )
 
@@ -358,6 +362,7 @@ class Pusher(object):
                     self.store.update_pusher_last_token(
                         self.app_id,
                         self.pushkey,
+                        self.user_name,
                         self.last_token
                     )
 
@@ -365,6 +370,7 @@ class Pusher(object):
                     self.store.update_pusher_failing_since(
                         self.app_id,
                         self.pushkey,
+                        self.user_name,
                         self.failing_since
                     )
                 else:
diff --git a/synapse/push/pusherpool.py b/synapse/push/pusherpool.py
index f75eebf8bf..cda072839c 100644
--- a/synapse/push/pusherpool.py
+++ b/synapse/push/pusherpool.py
@@ -84,6 +84,21 @@ class PusherPool:
             pushkey, lang, data
         )
 
+    @defer.inlineCallbacks
+    def remove_pushers_by_app_id_and_pushkey_not_user(self, app_id, pushkey,
+                                                      not_user_id):
+        to_remove = yield self.store.get_pushers_by_app_id_and_pushkey(
+            app_id, pushkey
+        )
+        for p in to_remove:
+            if p['user_name'] != not_user_id:
+                logger.info(
+                    "Removing pusher for app id %s, pushkey %s, user %s",
+                    app_id, pushkey, p['user_name']
+                )
+                self.remove_pusher(p['app_id'], p['pushkey'], p['user_name'])
+
+
     @defer.inlineCallbacks
     def _add_pusher_to_store(self, user_name, access_token, profile_tag, kind,
                              app_id, app_display_name, device_display_name,
@@ -101,7 +116,7 @@ class PusherPool:
             lang=lang,
             data=encode_canonical_json(data).decode("UTF-8"),
         )
-        self._refresh_pusher((app_id, pushkey))
+        self._refresh_pusher(app_id, pushkey, user_name)
 
     def _create_pusher(self, pusherdict):
         if pusherdict['kind'] == 'http':
@@ -126,30 +141,42 @@ class PusherPool:
             )
 
     @defer.inlineCallbacks
-    def _refresh_pusher(self, app_id_pushkey):
-        p = yield self.store.get_pushers_by_app_id_and_pushkey(
-            app_id_pushkey
+    def _refresh_pusher(self, app_id, pushkey, user_name):
+        resultlist = yield self.store.get_pushers_by_app_id_and_pushkey(
+            app_id, pushkey
         )
-        p['data'] = json.loads(p['data'])
+        p = None
+        for r in resultlist:
+            if r['user_name'] == user_name:
+                p = r
 
-        self._start_pushers([p])
+        if p:
+            p['data'] = json.loads(p['data'])
+
+            self._start_pushers([p])
 
     def _start_pushers(self, pushers):
         logger.info("Starting %d pushers", len(pushers))
         for pusherdict in pushers:
             p = self._create_pusher(pusherdict)
             if p:
-                fullid = "%s:%s" % (pusherdict['app_id'], pusherdict['pushkey'])
+                fullid = "%s:%s:%s" % (
+                    pusherdict['app_id'],
+                    pusherdict['pushkey'],
+                    pusherdict['user_name']
+                )
                 if fullid in self.pushers:
                     self.pushers[fullid].stop()
                 self.pushers[fullid] = p
                 p.start()
 
     @defer.inlineCallbacks
-    def remove_pusher(self, app_id, pushkey):
-        fullid = "%s:%s" % (app_id, pushkey)
+    def remove_pusher(self, app_id, pushkey, user_name):
+        fullid = "%s:%s:%s" % (app_id, pushkey, user_name)
         if fullid in self.pushers:
             logger.info("Stopping pusher %s", fullid)
             self.pushers[fullid].stop()
             del self.pushers[fullid]
-        yield self.store.delete_pusher_by_app_id_pushkey(app_id, pushkey)
+        yield self.store.delete_pusher_by_app_id_pushkey_user_name(
+            app_id, pushkey, user_name
+        )
diff --git a/synapse/rest/client/v1/pusher.py b/synapse/rest/client/v1/pusher.py
index 87e89c9305..c83287c028 100644
--- a/synapse/rest/client/v1/pusher.py
+++ b/synapse/rest/client/v1/pusher.py
@@ -37,7 +37,7 @@ class PusherRestServlet(ClientV1RestServlet):
                 and 'kind' in content and
                 content['kind'] is None):
             yield pusher_pool.remove_pusher(
-                content['app_id'], content['pushkey']
+                content['app_id'], content['pushkey'], user_name=user.to_string()
             )
             defer.returnValue((200, {}))
 
@@ -51,6 +51,17 @@ class PusherRestServlet(ClientV1RestServlet):
             raise SynapseError(400, "Missing parameters: "+','.join(missing),
                                errcode=Codes.MISSING_PARAM)
 
+        append = False
+        if 'append' in content:
+            append = content['append']
+
+        if not append:
+            yield pusher_pool.remove_pushers_by_app_id_and_pushkey_not_user(
+                app_id=content['app_id'],
+                pushkey=content['pushkey'],
+                not_user_id=user.to_string()
+            )
+
         try:
             yield pusher_pool.add_pusher(
                 user_name=user.to_string(),
diff --git a/synapse/storage/pusher.py b/synapse/storage/pusher.py
index 1ef8e06ac6..423878c6a0 100644
--- a/synapse/storage/pusher.py
+++ b/synapse/storage/pusher.py
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 
 class PusherStore(SQLBaseStore):
     @defer.inlineCallbacks
-    def get_pushers_by_app_id_and_pushkey(self, app_id_and_pushkey):
+    def get_pushers_by_app_id_and_pushkey(self, app_id, pushkey):
         sql = (
             "SELECT id, user_name, kind, profile_tag, app_id,"
             "app_display_name, device_display_name, pushkey, ts, data, "
@@ -38,7 +38,7 @@ class PusherStore(SQLBaseStore):
 
         rows = yield self._execute(
             "get_pushers_by_app_id_and_pushkey", None, sql,
-            app_id_and_pushkey[0], app_id_and_pushkey[1]
+            app_id, pushkey
         )
 
         ret = [
@@ -60,7 +60,7 @@ class PusherStore(SQLBaseStore):
             for r in rows
         ]
 
-        defer.returnValue(ret[0])
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_all_pushers(self):
@@ -104,9 +104,9 @@ class PusherStore(SQLBaseStore):
                 dict(
                     app_id=app_id,
                     pushkey=pushkey,
+                    user_name=user_name,
                 ),
                 dict(
-                    user_name=user_name,
                     access_token=access_token,
                     kind=kind,
                     profile_tag=profile_tag,
@@ -123,37 +123,38 @@ class PusherStore(SQLBaseStore):
             raise StoreError(500, "Problem creating pusher.")
 
     @defer.inlineCallbacks
-    def delete_pusher_by_app_id_pushkey(self, app_id, pushkey):
+    def delete_pusher_by_app_id_pushkey_user_name(self, app_id, pushkey, user_name):
         yield self._simple_delete_one(
             PushersTable.table_name,
-            {"app_id": app_id, "pushkey": pushkey},
-            desc="delete_pusher_by_app_id_pushkey",
+            {"app_id": app_id, "pushkey": pushkey, 'user_name': user_name},
+            desc="delete_pusher_by_app_id_pushkey_user_name",
         )
 
     @defer.inlineCallbacks
-    def update_pusher_last_token(self, app_id, pushkey, last_token):
+    def update_pusher_last_token(self, app_id, pushkey, user_name, last_token):
         yield self._simple_update_one(
             PushersTable.table_name,
-            {'app_id': app_id, 'pushkey': pushkey},
+            {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name},
             {'last_token': last_token},
             desc="update_pusher_last_token",
         )
 
     @defer.inlineCallbacks
-    def update_pusher_last_token_and_success(self, app_id, pushkey,
+    def update_pusher_last_token_and_success(self, app_id, pushkey, user_name,
                                              last_token, last_success):
         yield self._simple_update_one(
             PushersTable.table_name,
-            {'app_id': app_id, 'pushkey': pushkey},
+            {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name},
             {'last_token': last_token, 'last_success': last_success},
             desc="update_pusher_last_token_and_success",
         )
 
     @defer.inlineCallbacks
-    def update_pusher_failing_since(self, app_id, pushkey, failing_since):
+    def update_pusher_failing_since(self, app_id, pushkey, user_name,
+                                    failing_since):
         yield self._simple_update_one(
             PushersTable.table_name,
-            {'app_id': app_id, 'pushkey': pushkey},
+            {'app_id': app_id, 'pushkey': pushkey, 'user_name': user_name},
             {'failing_since': failing_since},
             desc="update_pusher_failing_since",
         )
diff --git a/synapse/storage/schema/delta/15/v15.sql b/synapse/storage/schema/delta/15/v15.sql
index fc3e436877..f5b2a08ca4 100644
--- a/synapse/storage/schema/delta/15/v15.sql
+++ b/synapse/storage/schema/delta/15/v15.sql
@@ -1,2 +1,25 @@
-ALTER TABLE pushers ADD COLUMN access_token INTEGER DEFAULT NULL;
-
+-- Drop, copy & recreate pushers table to change unique key
+-- Also add access_token column at the same time
+CREATE TABLE IF NOT EXISTS pushers2 (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  user_name TEXT NOT NULL,
+  access_token INTEGER DEFAULT NULL,
+  profile_tag varchar(32) NOT NULL,
+  kind varchar(8) NOT NULL,
+  app_id varchar(64) NOT NULL,
+  app_display_name varchar(64) NOT NULL,
+  device_display_name varchar(128) NOT NULL,
+  pushkey blob NOT NULL,
+  ts BIGINT NOT NULL,
+  lang varchar(8),
+  data blob,
+  last_token TEXT,
+  last_success BIGINT,
+  failing_since BIGINT,
+  FOREIGN KEY(user_name) REFERENCES users(name),
+  UNIQUE (app_id, pushkey, user_name)
+);
+INSERT INTO pushers2 (id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since)
+  SELECT id, user_name, profile_tag, kind, app_id, app_display_name, device_display_name, pushkey, ts, lang, data, last_token, last_success, failing_since FROM pushers;
+DROP TABLE pushers;
+ALTER TABLE pushers2 RENAME TO pushers;
-- 
cgit 1.5.1


From 9236136f3a4f0d8119d4a6333f37378f8e259e4a Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 1 Apr 2015 14:12:33 +0100
Subject: Make work in both Maria and SQLite. Fix tests

---
 synapse/app/homeserver.py                          | 20 ++++++----
 synapse/storage/__init__.py                        | 43 +++++++++++-----------
 synapse/storage/_base.py                           | 30 ++++++++++-----
 synapse/storage/engines/__init__.py                | 35 ++++++++++++++++++
 synapse/storage/engines/maria.py                   | 30 +++++++++++++++
 synapse/storage/engines/sqlite3.py                 | 25 +++++++++++++
 synapse/storage/keys.py                            |  4 +-
 synapse/storage/registration.py                    |  8 +++-
 synapse/storage/schema/delta/12/v12.sql            |  8 ++--
 synapse/storage/schema/delta/13/v13.sql            |  4 +-
 synapse/storage/schema/delta/14/v14.sql            |  2 +-
 .../storage/schema/full_schemas/11/event_edges.sql | 14 +++----
 .../schema/full_schemas/11/event_signatures.sql    |  8 ++--
 synapse/storage/schema/full_schemas/11/im.sql      | 18 ++++-----
 synapse/storage/schema/full_schemas/11/keys.sql    |  4 +-
 .../schema/full_schemas/11/media_repository.sql    |  8 ++--
 .../storage/schema/full_schemas/11/presence.sql    |  6 +--
 .../storage/schema/full_schemas/11/profiles.sql    |  4 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  2 +-
 .../schema/full_schemas/11/room_aliases.sql        |  8 ++--
 synapse/storage/schema/full_schemas/11/state.sql   |  6 +--
 .../schema/full_schemas/11/transactions.sql        |  8 ++--
 synapse/storage/schema/full_schemas/11/users.sql   | 10 ++---
 synapse/storage/signatures.py                      |  8 ++--
 synapse/storage/stream.py                          |  6 ---
 synapse/util/retryutils.py                         |  2 +-
 tests/federation/test_federation.py                | 10 +++--
 tests/handlers/test_federation.py                  |  9 +++++
 tests/handlers/test_presence.py                    |  7 +++-
 tests/handlers/test_typing.py                      |  7 +++-
 tests/rest/client/v1/test_events.py                |  9 -----
 tests/storage/test_appservice.py                   | 12 ++----
 tests/storage/test_base.py                         | 20 +++++-----
 tests/storage/test_registration.py                 | 36 +++++++++++-------
 tests/storage/test_roommember.py                   | 14 +++----
 tests/utils.py                                     | 11 +++++-
 36 files changed, 296 insertions(+), 160 deletions(-)
 create mode 100644 synapse/storage/engines/__init__.py
 create mode 100644 synapse/storage/engines/maria.py
 create mode 100644 synapse/storage/engines/sqlite3.py

(limited to 'synapse/storage/schema')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index beab6ffc7a..b185b2f569 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -20,6 +20,7 @@ sys.dont_write_bytecode = True
 from synapse.storage import (
     prepare_database, prepare_sqlite3_database, UpgradeDatabaseException,
 )
+from synapse.storage.engines import create_engine
 
 from synapse.server import HomeServer
 
@@ -376,7 +377,7 @@ def setup(config_options):
     if name in ["MySQLdb", "mysql.connector"]:
         db_config.setdefault("args", {}).update({
             "sql_mode": "TRADITIONAL",
-            "charset": "utf8",
+            "charset": "utf8mb4",
             "use_unicode": True,
         })
     elif name == "sqlite3":
@@ -388,6 +389,8 @@ def setup(config_options):
     else:
         raise RuntimeError("Unsupported database type '%s'" % (name,))
 
+    database_engine = create_engine(name)
+
     hs = SynapseHomeServer(
         config.server_name,
         domain_with_port=domain_with_port,
@@ -398,6 +401,7 @@ def setup(config_options):
         config=config,
         content_addr=config.content_addr,
         version_string=version_string,
+        database_engine=database_engine,
     )
 
     hs.create_resource_tree(
@@ -409,12 +413,14 @@ def setup(config_options):
     logger.info("Preparing database: %s...", db_name)
 
     try:
-        # with sqlite3.connect(db_name) as db_conn:
-        #     prepare_sqlite3_database(db_conn)
-        #     prepare_database(db_conn)
-        import mysql.connector
-        db_conn = mysql.connector.connect(**db_config.get("args", {}))
-        prepare_database(db_conn)
+        db_conn = database_engine.module.connect(**db_config.get("args", {}))
+
+        if name == "sqlite3":
+            prepare_sqlite3_database(db_conn)
+
+        prepare_database(db_conn, database_engine)
+
+        db_conn.commit()
     except UpgradeDatabaseException:
         sys.stderr.write(
             "\nFailed to upgrade database.\n"
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index abde7d0df5..f8053484cf 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -77,9 +77,6 @@ class DataStore(RoomMemberStore, RoomStore,
         self.min_token_deferred = self._get_min_token()
         self.min_token = None
 
-        self._next_stream_id_lock = threading.Lock()
-        self._next_stream_id = int(hs.get_clock().time_msec()) * 1000
-
     def insert_client_ip(self, user, access_token, device_id, ip, user_agent):
         return self._simple_upsert(
             "user_ips",
@@ -127,19 +124,21 @@ class UpgradeDatabaseException(PrepareDatabaseException):
     pass
 
 
-def prepare_database(db_conn):
+def prepare_database(db_conn, database_engine):
     """Prepares a database for usage. Will either create all necessary tables
     or upgrade from an older schema version.
     """
     try:
         cur = db_conn.cursor()
-        version_info = _get_or_create_schema_state(cur)
+        version_info = _get_or_create_schema_state(cur, database_engine)
 
         if version_info:
             user_version, delta_files, upgraded = version_info
-            _upgrade_existing_database(cur, user_version, delta_files, upgraded)
+            _upgrade_existing_database(
+                cur, user_version, delta_files, upgraded, database_engine
+            )
         else:
-            _setup_new_database(cur)
+            _setup_new_database(cur, database_engine)
 
         # cur.execute("PRAGMA user_version = %d" % (SCHEMA_VERSION,))
 
@@ -150,7 +149,7 @@ def prepare_database(db_conn):
         raise
 
 
-def _setup_new_database(cur):
+def _setup_new_database(cur, database_engine):
     """Sets up the database by finding a base set of "full schemas" and then
     applying any necessary deltas.
 
@@ -210,7 +209,7 @@ def _setup_new_database(cur):
         executescript(cur, sql_loc)
 
     cur.execute(
-        _convert_param_style(
+        database_engine.convert_param_style(
             "REPLACE INTO schema_version (version, upgraded)"
             " VALUES (?,?)"
         ),
@@ -221,12 +220,13 @@ def _setup_new_database(cur):
         cur,
         current_version=max_current_ver,
         applied_delta_files=[],
-        upgraded=False
+        upgraded=False,
+        database_engine=database_engine,
     )
 
 
 def _upgrade_existing_database(cur, current_version, applied_delta_files,
-                               upgraded):
+                               upgraded, database_engine):
     """Upgrades an existing database.
 
     Delta files can either be SQL stored in *.sql files, or python modules
@@ -335,26 +335,22 @@ def _upgrade_existing_database(cur, current_version, applied_delta_files,
 
             # Mark as done.
             cur.execute(
-                _convert_param_style(
+                database_engine.convert_param_style(
                     "INSERT INTO applied_schema_deltas (version, file)"
-                    " VALUES (?,?)"
+                    " VALUES (?,?)",
                 ),
                 (v, relative_path)
             )
 
             cur.execute(
-                _convert_param_style(
+                database_engine.convert_param_style(
                     "REPLACE INTO schema_version (version, upgraded)"
-                    " VALUES (?,?)"
+                    " VALUES (?,?)",
                 ),
                 (v, True)
             )
 
 
-def _convert_param_style(sql):
-    return sql.replace("?", "%s")
-
-
 def get_statements(f):
     statement_buffer = ""
     in_comment = False  # If we're in a /* ... */ style comment
@@ -409,7 +405,7 @@ def executescript(txn, schema_path):
             txn.execute(statement)
 
 
-def _get_or_create_schema_state(txn):
+def _get_or_create_schema_state(txn, database_engine):
     try:
         # Bluntly try creating the schema_version tables.
         schema_path = os.path.join(
@@ -426,7 +422,7 @@ def _get_or_create_schema_state(txn):
 
     if current_version:
         txn.execute(
-            _convert_param_style(
+            database_engine.convert_param_style(
                 "SELECT file FROM applied_schema_deltas WHERE version >= ?"
             ),
             (current_version,)
@@ -446,6 +442,8 @@ def prepare_sqlite3_database(db_conn):
     new. This only affects sqlite databases since they were the only ones
     supported at the time.
     """
+    import sqlite3
+
     with db_conn:
         schema_path = os.path.join(
             dir_path, "schema", "schema_version.sql",
@@ -466,7 +464,8 @@ def prepare_sqlite3_database(db_conn):
                 db_conn.execute(
                     _convert_param_style(
                         "REPLACE INTO schema_version (version, upgraded)"
-                        " VALUES (?,?)"
+                        " VALUES (?,?)",
+                        sqlite3
                     ),
                     (row[0], False)
                 )
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 76ec3ee93f..047d100f46 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -29,6 +29,7 @@ import functools
 import simplejson as json
 import sys
 import time
+import threading
 
 
 logger = logging.getLogger(__name__)
@@ -118,19 +119,16 @@ def cached(max_entries=1000, num_args=1):
     return wrap
 
 
-def _convert_param_style(sql):
-    return sql.replace("?", "%s")
-
-
 class LoggingTransaction(object):
     """An object that almost-transparently proxies for the 'txn' object
     passed to the constructor. Adds logging and metrics to the .execute()
     method."""
-    __slots__ = ["txn", "name"]
+    __slots__ = ["txn", "name", "database_engine"]
 
-    def __init__(self, txn, name):
+    def __init__(self, txn, name, database_engine):
         object.__setattr__(self, "txn", txn)
         object.__setattr__(self, "name", name)
+        object.__setattr__(self, "database_engine", database_engine)
 
     def __getattr__(self, name):
         return getattr(self.txn, name)
@@ -142,7 +140,7 @@ class LoggingTransaction(object):
         # TODO(paul): Maybe use 'info' and 'debug' for values?
         sql_logger.debug("[SQL] {%s} %s", self.name, sql)
 
-        sql = _convert_param_style(sql)
+        sql = self.database_engine.convert_param_style(sql)
 
         try:
             if args and args[0]:
@@ -227,9 +225,14 @@ class SQLBaseStore(object):
 
         self._get_event_cache = LruCache(hs.config.event_cache_size)
 
+        self.database_engine = hs.database_engine
+
         # Pretend the getEventCache is just another named cache
         caches_by_name["*getEvent*"] = self._get_event_cache
 
+        self._next_stream_id_lock = threading.Lock()
+        self._next_stream_id = int(hs.get_clock().time_msec()) * 1000
+
     def start_profiling(self):
         self._previous_loop_ts = self._clock.time_msec()
 
@@ -281,7 +284,10 @@ class SQLBaseStore(object):
                 sql_scheduling_timer.inc_by(time.time() * 1000 - start_time)
                 transaction_logger.debug("[TXN START] {%s}", name)
                 try:
-                    return func(LoggingTransaction(txn, name), *args, **kwargs)
+                    return func(
+                        LoggingTransaction(txn, name, self.database_engine),
+                        *args, **kwargs
+                    )
                 except:
                     logger.exception("[TXN FAIL] {%s}", name)
                     raise
@@ -588,7 +594,7 @@ class SQLBaseStore(object):
         select_sql = "SELECT %s FROM %s WHERE %s" % (
             ", ".join(retcols),
             table,
-            " AND ".join("%s = ?" % (k) for k in keyvalues)
+            " AND ".join("%s = ?" % (k,) for k in keyvalues)
         )
 
         txn.execute(select_sql, keyvalues.values())
@@ -836,6 +842,12 @@ class SQLBaseStore(object):
         result = txn.fetchone()
         return result[0] if result else None
 
+    def get_next_stream_id(self):
+        with self._next_stream_id_lock:
+            i = self._next_stream_id
+            self._next_stream_id += 1
+            return i
+
 
 class _RollbackButIsFineException(Exception):
     """ This exception is used to rollback a transaction without implying
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
new file mode 100644
index 0000000000..709b6f88ac
--- /dev/null
+++ b/synapse/storage/engines/__init__.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .maria import MariaEngine
+from .sqlite3 import Sqlite3Engine
+
+
+SUPPORTED_MODULE = {
+    "sqlite3": Sqlite3Engine,
+    "mysql.connector": MariaEngine,
+}
+
+
+def create_engine(name):
+    engine_class = SUPPORTED_MODULE.get(name, None)
+
+    if engine_class:
+        module = __import__(name)
+        return engine_class(module)
+
+    raise RuntimeError(
+        "Unsupported database engine '%s'" % (name,)
+    )
diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py
new file mode 100644
index 0000000000..df47763647
--- /dev/null
+++ b/synapse/storage/engines/maria.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import types
+
+
+class MariaEngine(object):
+    def __init__(self, database_module):
+        self.module = database_module
+
+    def convert_param_style(self, sql):
+        return sql.replace("?", "%s")
+
+    def encode_parameter(self, param):
+        if isinstance(param, types.BufferType):
+            return str(param)
+        return param
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
new file mode 100644
index 0000000000..639cdea41d
--- /dev/null
+++ b/synapse/storage/engines/sqlite3.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class Sqlite3Engine(object):
+    def __init__(self, database_module):
+        self.module = database_module
+
+    def convert_param_style(self, sql):
+        return sql
+
+    def encode_parameter(self, param):
+        return param
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index e6975a945b..25fef79434 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -64,7 +64,7 @@ class KeyStore(SQLBaseStore):
                 "fingerprint": fingerprint,
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "tls_certificate": tls_certificate_bytes,
+                "tls_certificate": buffer(tls_certificate_bytes),
             },
         )
 
@@ -113,6 +113,6 @@ class KeyStore(SQLBaseStore):
                 "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
-                "verify_key": verify_key.encode(),
+                "verify_key": buffer(verify_key.encode()),
             },
         )
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 7258f7b2a5..0c785ec989 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -42,6 +42,7 @@ class RegistrationStore(SQLBaseStore):
         yield self._simple_insert(
             "access_tokens",
             {
+                "id": self.get_next_stream_id(),
                 "user_id": user_id,
                 "token": token
             },
@@ -78,8 +79,11 @@ class RegistrationStore(SQLBaseStore):
 
         # it's possible for this to get a conflict, but only for a single user
         # since tokens are namespaced based on their user ID
-        txn.execute("INSERT INTO access_tokens(user_id, token) " +
-                    "VALUES (?,?)", [user_id, token])
+        txn.execute(
+            "INSERT INTO access_tokens(id, user_id, token)"
+            " VALUES (?,?,?)",
+            (self.get_next_stream_id(), user_id, token,)
+        )
 
     @defer.inlineCallbacks
     def get_user_by_id(self, user_id):
diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index b526109e6e..90ac474859 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections(
     reason VARCHAR(255) NOT NULL,
     last_check VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers (
   last_success BIGINT,
   failing_since BIGINT,
   UNIQUE (app_id, pushkey)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id BIGINT PRIMARY KEY,
@@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules (
   conditions VARCHAR(255) NOT NULL,
   actions VARCHAR(255) NOT NULL,
   UNIQUE(user_name, rule_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
@@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(255),
   filter_id BIGINT,
   filter_json BLOB
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
     user_id, filter_id
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index f0a5daf445..4953b6323c 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services(
     hs_token VARCHAR(255),
     sender VARCHAR(255),
     UNIQUE(token)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
     id BIGINT PRIMARY KEY,
@@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(255),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index a1260c5c1f..3bda073c94 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable (
   rule_id VARCHAR(255) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 0f53488e92..336cd563df 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges(
     room_id VARCHAR(255) NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
 CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
@@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth(
     room_id VARCHAR(255) NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
@@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations(
     destination VARCHAR(255) NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
 
@@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     type VARCHAR(255) NOT NULL,
     state_key VARCHAR(255) NOT NULL,
     UNIQUE (event_id, room_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
@@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth(
     auth_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
 CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 334d7c8680..11e611598b 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes (
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
 
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes (
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
 
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     key_id VARCHAR(255),
     signature BLOB,
     UNIQUE (event_id, signature_name, key_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
@@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes(
     algorithm VARCHAR(255),
     hash BLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 9849e969be..a0fb337629 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events(
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
 CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json(
     internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events(
     state_key VARCHAR(255) NOT NULL,
     prev_state VARCHAR(255),
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
 CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
@@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     state_key VARCHAR(255) NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
 CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
@@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     room_id VARCHAR(255) NOT NULL,
     membership VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
@@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback(
     sender VARCHAR(255),
     room_id VARCHAR(255),
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     topic VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
@@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms(
     room_id VARCHAR(255) PRIMARY KEY NOT NULL,
     is_public BOOL,
     creator VARCHAR(255)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS room_hosts(
     room_id VARCHAR(255) NOT NULL,
     host VARCHAR(255) NOT NULL,
     UNIQUE (room_id, host)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index c0f2ec29bb..a785cdb4c5 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(255), -- Server name.
@@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   ts_added_ms BIGINT, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index d9559f5902..27fe297af6 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     upload_name VARCHAR(255), -- The name the media was uploaded with.
     user_id VARCHAR(255), -- The user who uploaded the file.
     UNIQUE (media_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     media_id VARCHAR(255), -- The id used to refer to the media.
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
     )
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
@@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(255), -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
     media_origin VARCHAR(255), -- The remote HS the media came from.
@@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
      )
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
     ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index 8031321083..b48b110ae9 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence(
   status_msg VARCHAR(255),
   mtime BIGINT, -- miliseconds since last state change
   UNIQUE(user_id)
-) ENGINE = INNODB;
+) ;
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound(
   observed_user_id VARCHAR(255) NOT NULL,
   observer_user_id VARCHAR(255), -- a UserID,
   UNIQUE(observed_user_id)
-) ENGINE = INNODB;
+) ;
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
@@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list(
   observed_user_id VARCHAR(255), -- a UserID,
   accepted BOOLEAN,
   UNIQUE(user_id)
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 552645c56f..92da48f97e 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -14,7 +14,7 @@
  */
 CREATE TABLE IF NOT EXISTS profiles(
     user_id VARCHAR(255) NOT NULL,
-    displayname VARBINARY(255),
+    displayname VARCHAR(255),
     avatar_url VARCHAR(255),
     UNIQUE(user_id)
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index ba93e860f6..9b52a2012a 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions (
     event_id VARCHAR(255) NOT NULL,
     redacts VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
 CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 1e706aac2b..220df87573 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -14,12 +14,12 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias VARCHAR(255) NOT NULL,
+    room_alias VARBINARY(255) NOT NULL,
     room_id VARCHAR(255) NOT NULL,
     UNIQUE (room_alias)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias VARCHAR(255) NOT NULL,
+    room_alias VARBINARY(255) NOT NULL,
     server VARCHAR(255) NOT NULL
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index be9dc2920d..40584a325f 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups(
     id VARCHAR(20) PRIMARY KEY,
     room_id VARCHAR(255) NOT NULL,
     event_id VARCHAR(255) NOT NULL
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group VARCHAR(20) NOT NULL,
@@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state(
     type VARCHAR(255) NOT NULL,
     state_key VARCHAR(255) NOT NULL,
     event_id VARCHAR(255) NOT NULL
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
     event_id VARCHAR(255) NOT NULL,
     state_group VARCHAR(255) NOT NULL,
     UNIQUE (event_id)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index bd13bba8c2..d33bdfb301 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions(
     response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
     ts BIGINT
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
@@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     pdu_id VARCHAR(255),
     pdu_origin VARCHAR(255),
     UNIQUE (transaction_id, destination)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
@@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(255) PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
-) ENGINE = INNODB;
+) ;
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 55bffb22f3..28909f5805 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -14,20 +14,20 @@
  */
 CREATE TABLE IF NOT EXISTS users(
     name VARCHAR(255),
-    password_hash VARBINARY(255),
+    password_hash VARCHAR(255),
     creation_ts BIGINT,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id INTEGER PRIMARY KEY AUTO_INCREMENT,
+    id BIGINT PRIMARY KEY,
     user_id VARCHAR(255) NOT NULL,
     device_id VARCHAR(255),
     token VARCHAR(255) NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
-) ENGINE = INNODB;
+) ;
 
 CREATE TABLE IF NOT EXISTS user_ips (
     user VARCHAR(255) NOT NULL,
@@ -37,6 +37,6 @@ CREATE TABLE IF NOT EXISTS user_ips (
     user_agent VARCHAR(255) NOT NULL,
     last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
-) ENGINE = INNODB;
+) ;
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
diff --git a/synapse/storage/signatures.py b/synapse/storage/signatures.py
index 35bba854f9..f051828630 100644
--- a/synapse/storage/signatures.py
+++ b/synapse/storage/signatures.py
@@ -54,7 +54,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": hash_bytes,
+                "hash": buffer(hash_bytes),
             },
         )
 
@@ -116,7 +116,7 @@ class SignatureStore(SQLBaseStore):
             {
                 "event_id": event_id,
                 "algorithm": algorithm,
-                "hash": hash_bytes,
+                "hash": buffer(hash_bytes),
             },
         )
 
@@ -160,7 +160,7 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "signature_name": signature_name,
                 "key_id": key_id,
-                "signature": signature_bytes,
+                "signature": buffer(signature_bytes),
             },
         )
 
@@ -193,6 +193,6 @@ class SignatureStore(SQLBaseStore):
                 "event_id": event_id,
                 "prev_event_id": prev_event_id,
                 "algorithm": algorithm,
-                "hash": hash_bytes,
+                "hash": buffer(hash_bytes),
             },
         )
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 3a310cd003..e6bb5a8077 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -433,12 +433,6 @@ class StreamStore(SQLBaseStore):
 
         defer.returnValue(self.min_token)
 
-    def get_next_stream_id(self):
-        with self._next_stream_id_lock:
-            i = self._next_stream_id
-            self._next_stream_id += 1
-            return i
-
     def _get_room_events_max_id_txn(self, txn):
         txn.execute(
             "SELECT MAX(stream_ordering) as m FROM events"
diff --git a/synapse/util/retryutils.py b/synapse/util/retryutils.py
index 4e82232796..a42138f556 100644
--- a/synapse/util/retryutils.py
+++ b/synapse/util/retryutils.py
@@ -60,7 +60,7 @@ def get_retry_limiter(destination, clock, store, **kwargs):
 
     if retry_timings:
         retry_last_ts, retry_interval = (
-            retry_timings.retry_last_ts, retry_timings.retry_interval
+            retry_timings["retry_last_ts"], retry_timings["retry_interval"]
         )
 
         now = int(clock.time_msec())
diff --git a/tests/federation/test_federation.py b/tests/federation/test_federation.py
index 2ecd00d2ad..a4ef60b911 100644
--- a/tests/federation/test_federation.py
+++ b/tests/federation/test_federation.py
@@ -24,8 +24,6 @@ from ..utils import MockHttpResource, MockClock, setup_test_homeserver
 from synapse.federation import initialize_http_replication
 from synapse.events import FrozenEvent
 
-from synapse.storage.transactions import DestinationsTable
-
 
 def make_pdu(prev_pdus=[], **kwargs):
     """Provide some default fields for making a PduTuple."""
@@ -57,8 +55,14 @@ class FederationTestCase(unittest.TestCase):
         self.mock_persistence.get_received_txn_response.return_value = (
             defer.succeed(None)
         )
+
+        retry_timings_res = {
+            "destination": "",
+            "retry_last_ts": 0,
+            "retry_interval": 0,
+        }
         self.mock_persistence.get_destination_retry_timings.return_value = (
-            defer.succeed(DestinationsTable.EntryType("", 0, 0))
+            defer.succeed(retry_timings_res)
         )
         self.mock_persistence.get_auth_chain.return_value = []
         self.clock = MockClock()
diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py
index c13ade3286..08d2404b6c 100644
--- a/tests/handlers/test_federation.py
+++ b/tests/handlers/test_federation.py
@@ -87,6 +87,15 @@ class FederationTestCase(unittest.TestCase):
         self.datastore.get_room.return_value = defer.succeed(True)
         self.auth.check_host_in_room.return_value = defer.succeed(True)
 
+        retry_timings_res = {
+            "destination": "",
+            "retry_last_ts": 0,
+            "retry_interval": 0,
+        }
+        self.datastore.get_destination_retry_timings.return_value = (
+            defer.succeed(retry_timings_res)
+        )
+
         def have_events(event_ids):
             return defer.succeed({})
         self.datastore.have_events.side_effect = have_events
diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py
index 04eba4289e..9b0e606918 100644
--- a/tests/handlers/test_presence.py
+++ b/tests/handlers/test_presence.py
@@ -194,8 +194,13 @@ class MockedDatastorePresenceTestCase(PresenceTestCase):
         return datastore
 
     def setUp_datastore_federation_mocks(self, datastore):
+        retry_timings_res = {
+            "destination": "",
+            "retry_last_ts": 0,
+            "retry_interval": 0,
+        }
         datastore.get_destination_retry_timings.return_value = (
-            defer.succeed(DestinationsTable.EntryType("", 0, 0))
+            defer.succeed(retry_timings_res)
         )
 
         def get_received_txn_response(*args):
diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py
index bf34b7ccbd..2d76b23564 100644
--- a/tests/handlers/test_typing.py
+++ b/tests/handlers/test_typing.py
@@ -96,8 +96,13 @@ class TypingNotificationsTestCase(unittest.TestCase):
         self.event_source = hs.get_event_sources().sources["typing"]
 
         self.datastore = hs.get_datastore()
+        retry_timings_res = {
+            "destination": "",
+            "retry_last_ts": 0,
+            "retry_interval": 0,
+        }
         self.datastore.get_destination_retry_timings.return_value = (
-            defer.succeed(DestinationsTable.EntryType("", 0, 0))
+            defer.succeed(retry_timings_res)
         )
 
         def get_received_txn_response(*args):
diff --git a/tests/rest/client/v1/test_events.py b/tests/rest/client/v1/test_events.py
index 36b0f2ff6d..445272e323 100644
--- a/tests/rest/client/v1/test_events.py
+++ b/tests/rest/client/v1/test_events.py
@@ -115,12 +115,6 @@ class EventStreamPermissionsTestCase(RestTestCase):
         hs = yield setup_test_homeserver(
             http_client=None,
             replication_layer=Mock(),
-            clock=Mock(spec=[
-                "call_later",
-                "cancel_call_later",
-                "time_msec",
-                "time"
-            ]),
             ratelimiter=NonCallableMock(spec_set=[
                 "send_message",
             ]),
@@ -132,9 +126,6 @@ class EventStreamPermissionsTestCase(RestTestCase):
 
         hs.get_handlers().federation_handler = Mock()
 
-        hs.get_clock().time_msec.return_value = 1000000
-        hs.get_clock().time.return_value = 1000
-
         synapse.rest.client.v1.register.register_servlets(hs, self.mock_resource)
         synapse.rest.client.v1.events.register_servlets(hs, self.mock_resource)
         synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource)
diff --git a/tests/storage/test_appservice.py b/tests/storage/test_appservice.py
index ca5b92ec85..2ad55c8462 100644
--- a/tests/storage/test_appservice.py
+++ b/tests/storage/test_appservice.py
@@ -16,22 +16,18 @@ from tests import unittest
 from twisted.internet import defer
 
 from synapse.appservice import ApplicationService
-from synapse.server import HomeServer
 from synapse.storage.appservice import ApplicationServiceStore
 
-from mock import Mock
-from tests.utils import SQLiteMemoryDbPool, MockClock
+from tests.utils import setup_test_homeserver
 
 
 class ApplicationServiceStoreTestCase(unittest.TestCase):
 
     @defer.inlineCallbacks
     def setUp(self):
-        db_pool = SQLiteMemoryDbPool()
-        yield db_pool.prepare()
-        hs = HomeServer(
-            "test", db_pool=db_pool, clock=MockClock(), config=Mock()
-        )
+        hs = yield setup_test_homeserver()
+        db_pool = hs.get_db_pool()
+
         self.as_token = "token1"
         db_pool.runQuery(
             "INSERT INTO application_services(token) VALUES(?)",
diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py
index 7f5845cf0c..5c17d30148 100644
--- a/tests/storage/test_base.py
+++ b/tests/storage/test_base.py
@@ -24,6 +24,7 @@ from collections import OrderedDict
 from synapse.server import HomeServer
 
 from synapse.storage._base import SQLBaseStore
+from synapse.storage.engines import create_engine
 
 
 class SQLBaseStoreTestCase(unittest.TestCase):
@@ -40,7 +41,12 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         config = Mock()
         config.event_cache_size = 1
-        hs = HomeServer("test", db_pool=self.db_pool, config=config)
+        hs = HomeServer(
+            "test",
+            db_pool=self.db_pool,
+            config=config,
+            database_engine=create_engine("sqlite3"),
+        )
 
         self.datastore = SQLBaseStore(hs)
 
@@ -86,8 +92,7 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         self.assertEquals("Value", value)
         self.mock_txn.execute.assert_called_with(
-                "SELECT retcol FROM tablename WHERE keycol = ? "
-                "ORDER BY rowid asc",
+                "SELECT retcol FROM tablename WHERE keycol = ?",
                 ["TheKey"]
         )
 
@@ -104,8 +109,7 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         self.assertEquals({"colA": 1, "colB": 2, "colC": 3}, ret)
         self.mock_txn.execute.assert_called_with(
-                "SELECT colA, colB, colC FROM tablename WHERE keycol = ? "
-                "ORDER BY rowid asc",
+                "SELECT colA, colB, colC FROM tablename WHERE keycol = ?",
                 ["TheKey"]
         )
 
@@ -139,8 +143,7 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         self.assertEquals([{"colA": 1}, {"colA": 2}, {"colA": 3}], ret)
         self.mock_txn.execute.assert_called_with(
-                "SELECT colA FROM tablename WHERE keycol = ? "
-                "ORDER BY rowid asc",
+                "SELECT colA FROM tablename WHERE keycol = ?",
                 ["A set"]
         )
 
@@ -189,8 +192,7 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         self.assertEquals({"columname": "Old Value"}, ret)
         self.mock_txn.execute.assert_has_calls([
-                call('SELECT columname FROM tablename WHERE keycol = ? '
-                     'ORDER BY rowid asc',
+                call('SELECT columname FROM tablename WHERE keycol = ?',
                     ['TheKey']),
                 call("UPDATE tablename SET columname = ? WHERE keycol = ?",
                     ["New Value", "TheKey"])
diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py
index e0b81f2b57..78f6004204 100644
--- a/tests/storage/test_registration.py
+++ b/tests/storage/test_registration.py
@@ -42,28 +42,38 @@ class RegistrationStoreTestCase(unittest.TestCase):
         self.assertEquals(
             # TODO(paul): Surely this field should be 'user_id', not 'name'
             #  Additionally surely it shouldn't come in a 1-element list
-            [{"name": self.user_id, "password_hash": self.pwhash}],
+            {"name": self.user_id, "password_hash": self.pwhash},
             (yield self.store.get_user_by_id(self.user_id))
         )
 
-        self.assertEquals(
-            {"admin": 0,
-             "device_id": None,
-             "name": self.user_id,
-             "token_id": 1},
-            (yield self.store.get_user_by_token(self.tokens[0]))
+        result = yield self.store.get_user_by_token(self.tokens[1])
+
+        self.assertDictContainsSubset(
+            {
+                "admin": 0,
+                 "device_id": None,
+                 "name": self.user_id,
+            },
+            result
         )
 
+        self.assertTrue("token_id" in result)
+
     @defer.inlineCallbacks
     def test_add_tokens(self):
         yield self.store.register(self.user_id, self.tokens[0], self.pwhash)
         yield self.store.add_access_token_to_user(self.user_id, self.tokens[1])
 
-        self.assertEquals(
-            {"admin": 0,
-             "device_id": None,
-             "name": self.user_id,
-             "token_id": 2},
-            (yield self.store.get_user_by_token(self.tokens[1]))
+        result = yield self.store.get_user_by_token(self.tokens[1])
+
+        self.assertDictContainsSubset(
+            {
+                "admin": 0,
+                 "device_id": None,
+                 "name": self.user_id,
+            },
+            result
         )
 
+        self.assertTrue("token_id" in result)
+
diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py
index 811fea544b..785953cc89 100644
--- a/tests/storage/test_roommember.py
+++ b/tests/storage/test_roommember.py
@@ -119,7 +119,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
         yield self.inject_room_member(self.room, self.u_alice, Membership.JOIN)
 
         self.assertEquals(
-            ["test"],
+            {"test"},
             (yield self.store.get_joined_hosts_for_room(self.room.to_string()))
         )
 
@@ -127,7 +127,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
         yield self.inject_room_member(self.room, self.u_bob, Membership.JOIN)
 
         self.assertEquals(
-            ["test"],
+            {"test"},
             (yield self.store.get_joined_hosts_for_room(self.room.to_string()))
         )
 
@@ -136,9 +136,9 @@ class RoomMemberStoreTestCase(unittest.TestCase):
 
         self.assertEquals(
             {"test", "elsewhere"},
-            set((yield
+            (yield
                 self.store.get_joined_hosts_for_room(self.room.to_string())
-            ))
+            )
         )
 
         # Should still have both hosts
@@ -146,15 +146,15 @@ class RoomMemberStoreTestCase(unittest.TestCase):
 
         self.assertEquals(
             {"test", "elsewhere"},
-            set((yield
+            (yield
                 self.store.get_joined_hosts_for_room(self.room.to_string())
-            ))
+            )
         )
 
         # Should have only one host after other leaves
         yield self.inject_room_member(self.room, self.u_charlie, Membership.LEAVE)
 
         self.assertEquals(
-            ["test"],
+            {"test"},
             (yield self.store.get_joined_hosts_for_room(self.room.to_string()))
         )
diff --git a/tests/utils.py b/tests/utils.py
index 81e82a80df..cc038fecf1 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -17,6 +17,7 @@ from synapse.http.server import HttpServer
 from synapse.api.errors import cs_error, CodeMessageException, StoreError
 from synapse.api.constants import EventTypes
 from synapse.storage import prepare_database
+from synapse.storage.engines import create_engine
 from synapse.server import HomeServer
 
 from synapse.util.logcontext import LoggingContext
@@ -44,18 +45,23 @@ def setup_test_homeserver(name="test", datastore=None, config=None, **kargs):
         config.event_cache_size = 1
         config.disable_registration = False
 
+    if "clock" not in kargs:
+        kargs["clock"] = MockClock()
+
     if datastore is None:
         db_pool = SQLiteMemoryDbPool()
         yield db_pool.prepare()
         hs = HomeServer(
             name, db_pool=db_pool, config=config,
             version_string="Synapse/tests",
+            database_engine=create_engine("sqlite3"),
             **kargs
         )
     else:
         hs = HomeServer(
             name, db_pool=None, datastore=datastore, config=config,
             version_string="Synapse/tests",
+            database_engine=create_engine("sqlite3"),
             **kargs
         )
 
@@ -227,7 +233,10 @@ class SQLiteMemoryDbPool(ConnectionPool, object):
         )
 
     def prepare(self):
-        return self.runWithConnection(prepare_database)
+        engine = create_engine("sqlite3")
+        return self.runWithConnection(
+            lambda conn: prepare_database(conn, engine)
+        )
 
 
 class MemoryDataStore(object):
-- 
cgit 1.5.1


From 779f7b0f443bfab351702161763b62b7bb6a27d8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 2 Apr 2015 10:06:22 +0100
Subject: Fix unicode support

---
 synapse/app/homeserver.py                          | 12 ++--
 synapse/storage/_base.py                           | 22 +++----
 synapse/storage/engines/__init__.py                |  4 +-
 synapse/storage/engines/maria.py                   | 12 ++++
 synapse/storage/engines/sqlite3.py                 |  9 +++
 synapse/storage/keys.py                            | 14 +++--
 synapse/storage/schema/delta/12/v12.sql            | 18 +++---
 synapse/storage/schema/delta/13/v13.sql            | 10 ++--
 synapse/storage/schema/delta/14/v14.sql            |  4 +-
 .../storage/schema/full_schemas/11/event_edges.sql | 34 +++++------
 .../schema/full_schemas/11/event_signatures.sql    | 20 +++----
 synapse/storage/schema/full_schemas/11/im.sql      | 68 +++++++++++-----------
 synapse/storage/schema/full_schemas/11/keys.sql    | 12 ++--
 .../schema/full_schemas/11/media_repository.sql    | 34 +++++------
 .../storage/schema/full_schemas/11/presence.sql    | 16 +++--
 .../storage/schema/full_schemas/11/profiles.sql    |  6 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  4 +-
 .../schema/full_schemas/11/room_aliases.sql        |  8 +--
 synapse/storage/schema/full_schemas/11/state.sql   | 16 ++---
 .../schema/full_schemas/11/transactions.sql        | 16 ++---
 synapse/storage/schema/full_schemas/11/users.sql   | 20 +++----
 synapse/storage/schema/schema_version.sql          |  2 +-
 22 files changed, 193 insertions(+), 168 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index 1ab6effd5f..9d7a58080d 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -18,7 +18,7 @@ import sys
 sys.dont_write_bytecode = True
 
 from synapse.storage import (
-    prepare_database, prepare_sqlite3_database, UpgradeDatabaseException,
+    prepare_database, UpgradeDatabaseException,
 )
 from synapse.storage.engines import create_engine
 
@@ -381,19 +381,18 @@ def setup(config_options):
             "sql_mode": "TRADITIONAL",
             "charset": "utf8mb4",
             "use_unicode": True,
+            "collation": "utf8mb4_general_ci",
         })
     elif name == "sqlite3":
-        def open_fun(conn):
-            prepare_database(conn, database_engine)
         db_config.setdefault("args", {}).update({
             "cp_min": 1,
             "cp_max": 1,
-            "cp_openfun": open_fun,
         })
     else:
         raise RuntimeError("Unsupported database type '%s'" % (name,))
 
     database_engine = create_engine(name)
+    db_config["args"]["cp_openfun"] = database_engine.on_new_connection
 
     hs = SynapseHomeServer(
         config.server_name,
@@ -424,10 +423,7 @@ def setup(config_options):
             }
         )
 
-        if name == "sqlite3":
-            prepare_sqlite3_database(db_conn)
-
-        prepare_database(db_conn, database_engine)
+        database_engine.prepare_database(db_conn)
 
         db_conn.commit()
     except UpgradeDatabaseException:
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 047d100f46..de4f661973 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -142,19 +142,23 @@ class LoggingTransaction(object):
 
         sql = self.database_engine.convert_param_style(sql)
 
-        try:
-            if args and args[0]:
-                values = args[0]
+        if args and args[0]:
+            args = list(args)
+            args[0] = [
+                self.database_engine.encode_parameter(a) for a in args[0]
+            ]
+            try:
                 sql_logger.debug(
-                    "[SQL values] {%s} " + ", ".join(("<%r>",) * len(values)),
+                    "[SQL values] {%s} " + ", ".join(("<%r>",) * len(args[0])),
                     self.name,
-                    *values
+                    *args[0]
                 )
-        except:
-            # Don't let logging failures stop SQL from working
-            pass
+            except:
+                # Don't let logging failures stop SQL from working
+                pass
 
         start = time.time() * 1000
+
         try:
             return self.txn.execute(
                 sql, *args, **kwargs
@@ -761,8 +765,6 @@ class SQLBaseStore(object):
             return None
 
         internal_metadata, js, redacted, rejected_reason = res
-        js = js.decode("utf8")
-        internal_metadata = internal_metadata.decode("utf8")
 
         start_time = update_counter("select_event", start_time)
 
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 709b6f88ac..29702be923 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -16,6 +16,8 @@
 from .maria import MariaEngine
 from .sqlite3 import Sqlite3Engine
 
+import importlib
+
 
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
@@ -27,7 +29,7 @@ def create_engine(name):
     engine_class = SUPPORTED_MODULE.get(name, None)
 
     if engine_class:
-        module = __import__(name)
+        module = importlib.import_module(name)
         return engine_class(module)
 
     raise RuntimeError(
diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py
index df47763647..7fcb706a60 100644
--- a/synapse/storage/engines/maria.py
+++ b/synapse/storage/engines/maria.py
@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.storage import prepare_database
 
 import types
 
@@ -28,3 +29,14 @@ class MariaEngine(object):
         if isinstance(param, types.BufferType):
             return str(param)
         return param
+
+    def on_new_connection(self, db_conn):
+        pass
+
+    def prepare_database(self, db_conn):
+        cur = db_conn.cursor()
+        cur.execute(
+            "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci"
+        )
+        db_conn.commit()
+        prepare_database(db_conn, self)
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 639cdea41d..e802b5d5fd 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from synapse.storage import prepare_database, prepare_sqlite3_database
+
 
 class Sqlite3Engine(object):
     def __init__(self, database_module):
@@ -23,3 +25,10 @@ class Sqlite3Engine(object):
 
     def encode_parameter(self, param):
         return param
+
+    def on_new_connection(self, db_conn):
+        self.prepare_database(db_conn)
+
+    def prepare_database(self, db_conn):
+        prepare_sqlite3_database(db_conn)
+        prepare_database(db_conn, self)
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index 25fef79434..d3b9b38664 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -57,15 +57,18 @@ class KeyStore(SQLBaseStore):
             OpenSSL.crypto.FILETYPE_ASN1, tls_certificate
         )
         fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest()
-        return self._simple_insert(
+        return self._simple_upsert(
             table="server_tls_certificates",
-            values={
+            keyvalues={
                 "server_name": server_name,
                 "fingerprint": fingerprint,
+            },
+            values={
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
                 "tls_certificate": buffer(tls_certificate_bytes),
             },
+            desc="store_server_certificate",
         )
 
     @defer.inlineCallbacks
@@ -106,13 +109,16 @@ class KeyStore(SQLBaseStore):
             ts_now_ms (int): The time now in milliseconds
             verification_key (VerifyKey): The NACL verify key.
         """
-        return self._simple_insert(
+        return self._simple_upsert(
             table="server_signature_keys",
-            values={
+            keyvalues={
                 "server_name": server_name,
                 "key_id": "%s:%s" % (verify_key.alg, verify_key.version),
+            },
+            values={
                 "from_server": from_server,
                 "ts_added_ms": time_now_ms,
                 "verify_key": buffer(verify_key.encode()),
             },
+            desc="store_server_verify_key",
         )
diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 90ac474859..717d289f78 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -14,16 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS rejections(
-    event_id VARCHAR(255) NOT NULL,
-    reason VARCHAR(255) NOT NULL,
-    last_check VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    reason VARCHAR(150) NOT NULL,
+    last_check VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
   app_id VARCHAR(64) NOT NULL,
@@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers (
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
-  rule_id VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
   priority_class TINYINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
-  conditions VARCHAR(255) NOT NULL,
-  actions VARCHAR(255) NOT NULL,
+  conditions VARCHAR(150) NOT NULL,
+  actions VARCHAR(150) NOT NULL,
   UNIQUE(user_name, rule_id)
 ) ;
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
-  user_id VARCHAR(255),
+  user_id VARCHAR(150),
   filter_id BIGINT,
   filter_json BLOB
 ) ;
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 4953b6323c..f5275a59b6 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -15,10 +15,10 @@
 
 CREATE TABLE IF NOT EXISTS application_services(
     id BIGINT PRIMARY KEY,
-    url VARCHAR(255),
-    token VARCHAR(255),
-    hs_token VARCHAR(255),
-    sender VARCHAR(255),
+    url VARCHAR(150),
+    token VARCHAR(150),
+    hs_token VARCHAR(150),
+    sender VARCHAR(150),
     UNIQUE(token)
 ) ;
 
@@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     id BIGINT PRIMARY KEY,
     as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex VARCHAR(255),
+    regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
 ) ;
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index 3bda073c94..1d582cc626 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,7 +1,7 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(255) NOT NULL,
-  rule_id VARCHAR(255) NOT NULL,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 336cd563df..124c9a9bdf 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
-    event_id VARCHAR(255) NOT NULL,
-    prev_event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    prev_event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 ) ;
@@ -46,7 +46,7 @@ CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
-    room_id VARCHAR(255) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
 ) ;
@@ -55,8 +55,8 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
-    event_id VARCHAR(255) NOT NULL,
-    destination VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    destination VARCHAR(150) NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 ) ;
@@ -65,10 +65,10 @@ CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id)
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
 ) ;
 
@@ -79,9 +79,9 @@ CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
-    event_id VARCHAR(255) NOT NULL,
-    auth_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    auth_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 11e611598b..30e3f71c5f 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
 ) ;
@@ -24,8 +24,8 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
 ) ;
@@ -34,9 +34,9 @@ CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(e
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id VARCHAR(255),
-    signature_name VARCHAR(255),
-    key_id VARCHAR(255),
+    event_id VARCHAR(150),
+    signature_name VARCHAR(150),
+    key_id VARCHAR(150),
     signature BLOB,
     UNIQUE (event_id, signature_name, key_id)
 ) ;
@@ -45,9 +45,9 @@ CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id VARCHAR(255),
-    prev_event_id VARCHAR(255),
-    algorithm VARCHAR(255),
+    event_id VARCHAR(150),
+    prev_event_id VARCHAR(150),
+    algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index a0fb337629..7cb8f802e1 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -16,9 +16,9 @@
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering BIGINT PRIMARY KEY,
     topological_ordering BIGINT NOT NULL,
-    event_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     content BLOB NOT NULL,
     unrecognized_keys BLOB,
     processed BOOL NOT NULL,
@@ -33,8 +33,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
     UNIQUE (event_id)
@@ -44,11 +44,11 @@ CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
-    prev_state VARCHAR(255),
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    prev_state VARCHAR(150),
     UNIQUE (event_id)
 ) ;
 
@@ -58,10 +58,10 @@ CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
 ) ;
@@ -71,11 +71,11 @@ CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (ty
 CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id VARCHAR(255) NOT NULL,
-    user_id VARCHAR(255) NOT NULL,
-    sender VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    membership VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
+    sender VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    membership VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
@@ -83,41 +83,41 @@ CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
-    event_id VARCHAR(255) NOT NULL,
-    feedback_type VARCHAR(255),
-    target_event_id VARCHAR(255),
-    sender VARCHAR(255),
-    room_id VARCHAR(255),
+    event_id VARCHAR(150) NOT NULL,
+    feedback_type VARCHAR(150),
+    target_event_id VARCHAR(150),
+    sender VARCHAR(150),
+    room_id VARCHAR(150),
     UNIQUE (event_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS topics(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    topic VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    topic VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
-    event_id VARCHAR(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    name VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    name VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 );
 
 CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
-    room_id VARCHAR(255) PRIMARY KEY NOT NULL,
+    room_id VARCHAR(150) PRIMARY KEY NOT NULL,
     is_public BOOL,
-    creator VARCHAR(255)
+    creator VARCHAR(150)
 ) ;
 
 CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id VARCHAR(255) NOT NULL,
-    host VARCHAR(255) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    host VARCHAR(150) NOT NULL,
     UNIQUE (room_id, host)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index a785cdb4c5..062ca53fef 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -13,18 +13,18 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name VARCHAR(255), -- Server name.
-  fingerprint VARCHAR(255), -- Certificate fingerprint.
-  from_server VARCHAR(255), -- Which key server the certificate was fetched from.
+  server_name VARCHAR(150), -- Server name.
+  fingerprint VARCHAR(150), -- Certificate fingerprint.
+  from_server VARCHAR(150), -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 ) ;
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name VARCHAR(255), -- Server name.
-  key_id VARCHAR(255), -- Key version.
-  from_server VARCHAR(255), -- Which key server the key was fetched form.
+  server_name VARCHAR(150), -- Server name.
+  key_id VARCHAR(150), -- Key version.
+  from_server VARCHAR(150), -- Which key server the key was fetched form.
   ts_added_ms BIGINT, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 27fe297af6..c8c5f1d2f0 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -14,21 +14,21 @@
  */
 
 CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id VARCHAR(255), -- The id used to refer to the media.
-    media_type VARCHAR(255), -- The MIME-type of the media.
+    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(255), -- The name the media was uploaded with.
-    user_id VARCHAR(255), -- The user who uploaded the file.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id VARCHAR(255), -- The id used to refer to the media.
+    media_id VARCHAR(150), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
-    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail.
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
@@ -39,25 +39,25 @@ CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin VARCHAR(255), -- The remote HS the media came from.
-    media_id VARCHAR(255), -- The id used to refer to the media on that server.
-    media_type VARCHAR(255), -- The MIME-type of the media.
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media on that server.
+    media_type VARCHAR(150), -- The MIME-type of the media.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(255), -- The name the media was uploaded with.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
 ) ;
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin VARCHAR(255), -- The remote HS the media came from.
-    media_id VARCHAR(255), -- The id used to refer to the media.
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method VARCHAR(255), -- The method used to make the thumbnail
-    thumbnail_type VARCHAR(255), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id VARCHAR(255), -- The name used to store the media on disk.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
     UNIQUE (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index b48b110ae9..273e61281a 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -13,9 +13,9 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS presence(
-  user_id VARCHAR(255) NOT NULL,
+  user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
-  status_msg VARCHAR(255),
+  status_msg VARCHAR(150),
   mtime BIGINT, -- miliseconds since last state change
   UNIQUE(user_id)
 ) ;
@@ -23,16 +23,14 @@ CREATE TABLE IF NOT EXISTS presence(
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id VARCHAR(255) NOT NULL,
-  observer_user_id VARCHAR(255), -- a UserID,
-  UNIQUE(observed_user_id)
+  observed_user_id VARCHAR(150) NOT NULL,
+  observer_user_id VARCHAR(150) -- a UserID,
 ) ;
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
-  user_id VARCHAR(255) NOT NULL,
-  observed_user_id VARCHAR(255), -- a UserID,
-  accepted BOOLEAN,
-  UNIQUE(user_id)
+  user_id VARCHAR(150) NOT NULL,
+  observed_user_id VARCHAR(150), -- a UserID,
+  accepted BOOLEAN
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 92da48f97e..023060a54c 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS profiles(
-    user_id VARCHAR(255) NOT NULL,
-    displayname VARCHAR(255),
-    avatar_url VARCHAR(255),
+    user_id VARCHAR(150) NOT NULL,
+    displayname VARCHAR(150),
+    avatar_url VARCHAR(150),
     UNIQUE(user_id)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 9b52a2012a..5c23188d62 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS redactions (
-    event_id VARCHAR(255) NOT NULL,
-    redacts VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    redacts VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 220df87573..63fe0f5c64 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -14,12 +14,12 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias VARBINARY(255) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
+    room_alias VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
     UNIQUE (room_alias)
 ) ;
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias VARBINARY(255) NOT NULL,
-    server VARCHAR(255) NOT NULL
+    room_alias VARCHAR(150) NOT NULL,
+    server VARCHAR(150) NOT NULL
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 40584a325f..acfb76439b 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -15,21 +15,21 @@
 
 CREATE TABLE IF NOT EXISTS state_groups(
     id VARCHAR(20) PRIMARY KEY,
-    room_id VARCHAR(255) NOT NULL,
-    event_id VARCHAR(255) NOT NULL
+    room_id VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
 ) ;
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group VARCHAR(20) NOT NULL,
-    room_id VARCHAR(255) NOT NULL,
-    type VARCHAR(255) NOT NULL,
-    state_key VARCHAR(255) NOT NULL,
-    event_id VARCHAR(255) NOT NULL
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
 ) ;
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id VARCHAR(255) NOT NULL,
-    state_group VARCHAR(255) NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    state_group VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
 ) ;
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index d33bdfb301..43541661ce 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -14,8 +14,8 @@
  */
 -- Stores what transaction ids we have received and what our response was
 CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id VARCHAR(255),
-    origin VARCHAR(255),
+    transaction_id VARCHAR(150),
+    origin VARCHAR(150),
     ts BIGINT,
     response_code INTEGER,
     response_json BLOB,
@@ -30,8 +30,8 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
     id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
-    transaction_id VARCHAR(255),
-    destination VARCHAR(255),
+    transaction_id VARCHAR(150),
+    destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
     ts BIGINT
@@ -47,9 +47,9 @@ CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_c
 -- For sent transactions only.
 CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
-    destination VARCHAR(255),
-    pdu_id VARCHAR(255),
-    pdu_origin VARCHAR(255),
+    destination VARCHAR(150),
+    pdu_id VARCHAR(150),
+    pdu_origin VARCHAR(150),
     UNIQUE (transaction_id, destination)
 ) ;
 
@@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
-    destination VARCHAR(255) PRIMARY KEY,
+    destination VARCHAR(150) PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 28909f5805..0271de3526 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    name VARCHAR(255),
-    password_hash VARCHAR(255),
+    name VARCHAR(150),
+    password_hash VARCHAR(150),
     creation_ts BIGINT,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
@@ -22,19 +22,19 @@ CREATE TABLE IF NOT EXISTS users(
 
 CREATE TABLE IF NOT EXISTS access_tokens(
     id BIGINT PRIMARY KEY,
-    user_id VARCHAR(255) NOT NULL,
-    device_id VARCHAR(255),
-    token VARCHAR(255) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    token VARCHAR(150) NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
 ) ;
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user VARCHAR(255) NOT NULL,
-    access_token VARCHAR(255) NOT NULL,
-    device_id VARCHAR(255),
-    ip VARCHAR(255) NOT NULL,
-    user_agent VARCHAR(255) NOT NULL,
+    user VARCHAR(150) NOT NULL,
+    access_token VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    ip VARCHAR(150) NOT NULL,
+    user_agent VARCHAR(150) NOT NULL,
     last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
 ) ;
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index 28762861e5..e7fa6fe569 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS schema_version(
 
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
     `version` INTEGER NOT NULL,
-    `file` VARCHAR(255) NOT NULL,
+    `file` VARCHAR(150) NOT NULL,
     UNIQUE(version, file)
 );
 
-- 
cgit 1.5.1


From 0af5f5efaf0b24187514cf78d7982ef9b85a208c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Apr 2015 12:08:35 +0100
Subject: Don't use multiple UNIQUE constraints; it will cause deadlocks

---
 synapse/storage/_base.py                            |  4 ++--
 synapse/storage/events.py                           | 14 +++++++-------
 synapse/storage/schema/full_schemas/11/presence.sql | 10 ++++++----
 synapse/storage/schema/full_schemas/11/users.sql    |  4 ++--
 4 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index a0c1718c27..4ac61be895 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -837,11 +837,11 @@ class SQLBaseStore(object):
             return curr_time
 
         logger.debug("Got js: %r", js)
-        d = json.loads(js)
+        d = json.loads(str(js).decode("utf8"))
         start_time = update_counter("decode_json", start_time)
 
         logger.debug("Got internal_metadata: %r", internal_metadata)
-        internal_metadata = json.loads(internal_metadata)
+        internal_metadata = json.loads(str(internal_metadata).decode("utf8"))
         start_time = update_counter("decode_internal", start_time)
 
         ev = FrozenEvent(d, internal_metadata_dict=internal_metadata)
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 514feebcbf..3b3416716e 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -179,7 +179,7 @@ class EventsStore(SQLBaseStore):
                 )
                 txn.execute(
                     sql,
-                    (metadata_json.decode("UTF-8"), event.event_id,)
+                    (buffer(metadata_json), event.event_id,)
                 )
 
                 sql = (
@@ -224,14 +224,14 @@ class EventsStore(SQLBaseStore):
             values={
                 "event_id": event.event_id,
                 "room_id": event.room_id,
-                "internal_metadata": metadata_json.decode("UTF-8"),
-                "json": encode_canonical_json(event_dict).decode("UTF-8"),
+                "internal_metadata": buffer(metadata_json),
+                "json": buffer(encode_canonical_json(event_dict)),
             },
         )
 
-        content = encode_canonical_json(
+        content = buffer(encode_canonical_json(
             event.content
-        ).decode("UTF-8")
+        ))
 
         vals = {
             "topological_ordering": event.depth,
@@ -256,9 +256,9 @@ class EventsStore(SQLBaseStore):
             ]
         }
 
-        vals["unrecognized_keys"] = encode_canonical_json(
+        vals["unrecognized_keys"] = buffer(encode_canonical_json(
             unrec
-        ).decode("UTF-8")
+        ))
 
         sql = (
             "INSERT INTO events"
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index 273e61281a..00d803a5cd 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -17,20 +17,22 @@ CREATE TABLE IF NOT EXISTS presence(
   state VARCHAR(20),
   status_msg VARCHAR(150),
   mtime BIGINT, -- miliseconds since last state change
-  UNIQUE(user_id)
+  UNIQUE (user_id)
 ) ;
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
   observed_user_id VARCHAR(150) NOT NULL,
-  observer_user_id VARCHAR(150) -- a UserID,
+  observer_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  UNIQUE (observed_user_id, observer_user_id)
 ) ;
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
   user_id VARCHAR(150) NOT NULL,
-  observed_user_id VARCHAR(150), -- a UserID,
-  accepted BOOLEAN
+  observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  accepted BOOLEAN NOT NULL,
+  UNIQUE (user_id, observed_user_id)
 ) ;
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 0271de3526..ba0f42d455 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -35,8 +35,8 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT NOT NULL,
-    UNIQUE (user, access_token, ip, user_agent)
+    last_seen BIGINT NOT NULL
 ) ;
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
-- 
cgit 1.5.1


From 8574bf62dc27ec3c2059c57a32c955e85d7e51df Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 7 Apr 2015 12:09:36 +0100
Subject: Add index to presence table

---
 synapse/storage/schema/delta/15/presence_indices.sql | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 synapse/storage/schema/delta/15/presence_indices.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/15/presence_indices.sql b/synapse/storage/schema/delta/15/presence_indices.sql
new file mode 100644
index 0000000000..6b8d0f1ca7
--- /dev/null
+++ b/synapse/storage/schema/delta/15/presence_indices.sql
@@ -0,0 +1,2 @@
+
+CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id);
-- 
cgit 1.5.1


From 9707acfc40547277144438fd8f077bd3bcbf7001 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 9 Apr 2015 13:45:20 +0100
Subject: Remove spurious spaces

---
 synapse/storage/schema/delta/12/v12.sql                |  8 ++++----
 synapse/storage/schema/delta/13/v13.sql                |  4 ++--
 synapse/storage/schema/delta/14/v14.sql                |  2 +-
 synapse/storage/schema/full_schemas/11/event_edges.sql | 14 +++++++-------
 .../schema/full_schemas/11/event_signatures.sql        |  8 ++++----
 synapse/storage/schema/full_schemas/11/im.sql          | 18 +++++++++---------
 synapse/storage/schema/full_schemas/11/keys.sql        |  4 ++--
 .../schema/full_schemas/11/media_repository.sql        |  8 ++++----
 synapse/storage/schema/full_schemas/11/presence.sql    |  6 +++---
 synapse/storage/schema/full_schemas/11/profiles.sql    |  2 +-
 synapse/storage/schema/full_schemas/11/redactions.sql  |  2 +-
 .../storage/schema/full_schemas/11/room_aliases.sql    |  4 ++--
 synapse/storage/schema/full_schemas/11/state.sql       |  6 +++---
 .../storage/schema/full_schemas/11/transactions.sql    |  8 ++++----
 synapse/storage/schema/full_schemas/11/users.sql       |  6 +++---
 15 files changed, 50 insertions(+), 50 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 717d289f78..ed8a2f3f10 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS rejections(
     reason VARCHAR(150) NOT NULL,
     last_check VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS pushers (
   last_success BIGINT,
   failing_since BIGINT,
   UNIQUE (app_id, pushkey)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id BIGINT PRIMARY KEY,
@@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS push_rules (
   conditions VARCHAR(150) NOT NULL,
   actions VARCHAR(150) NOT NULL,
   UNIQUE(user_name, rule_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
@@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
   filter_id BIGINT,
   filter_json BLOB
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
     user_id, filter_id
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index f5275a59b6..ef6551f3ad 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS application_services(
     hs_token VARCHAR(150),
     sender VARCHAR(150),
     UNIQUE(token)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
     id BIGINT PRIMARY KEY,
@@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
-) ;
+);
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index 1d582cc626..f1208c731d 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -4,6 +4,6 @@ CREATE TABLE IF NOT EXISTS push_rules_enable (
   rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 124c9a9bdf..602c11411d 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
 CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_edges(
     room_id VARCHAR(150) NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
 CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
@@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS room_depth(
     room_id VARCHAR(150) NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 
@@ -59,7 +59,7 @@ create TABLE IF NOT EXISTS event_destinations(
     destination VARCHAR(150) NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
 
@@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     type VARCHAR(150) NOT NULL,
     state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id, room_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
@@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS event_auth(
     auth_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
 CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 30e3f71c5f..0c27bc1f34 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS event_content_hashes (
     algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
 
@@ -28,7 +28,7 @@ CREATE TABLE IF NOT EXISTS event_reference_hashes (
     algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, algorithm)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
 
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     key_id VARCHAR(150),
     signature BLOB,
     UNIQUE (event_id, signature_name, key_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
 
@@ -50,6 +50,6 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes(
     algorithm VARCHAR(150),
     hash BLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 7cb8f802e1..363e2c842e 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS events(
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
 CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
@@ -38,7 +38,7 @@ CREATE TABLE IF NOT EXISTS event_json(
     internal_metadata BLOB NOT NULL,
     json BLOB NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
 
@@ -50,7 +50,7 @@ CREATE TABLE IF NOT EXISTS state_events(
     state_key VARCHAR(150) NOT NULL,
     prev_state VARCHAR(150),
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
 CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
@@ -64,7 +64,7 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     state_key VARCHAR(150) NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
 CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
@@ -77,7 +77,7 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     room_id VARCHAR(150) NOT NULL,
     membership VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
@@ -89,14 +89,14 @@ CREATE TABLE IF NOT EXISTS feedback(
     sender VARCHAR(150),
     room_id VARCHAR(150),
     UNIQUE (event_id)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     topic VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
@@ -113,12 +113,12 @@ CREATE TABLE IF NOT EXISTS rooms(
     room_id VARCHAR(150) PRIMARY KEY NOT NULL,
     is_public BOOL,
     creator VARCHAR(150)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS room_hosts(
     room_id VARCHAR(150) NOT NULL,
     host VARCHAR(150) NOT NULL,
     UNIQUE (room_id, host)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 062ca53fef..0e80bf761e 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -19,7 +19,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(150), -- Server name.
@@ -28,4 +28,4 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   ts_added_ms BIGINT, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
-) ;
+);
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index c8c5f1d2f0..08c42722d2 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     media_id VARCHAR(150), -- The id used to refer to the media.
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
     )
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
@@ -47,7 +47,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(150), -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
     media_origin VARCHAR(150), -- The remote HS the media came from.
@@ -62,7 +62,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
      )
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
     ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index 00d803a5cd..f3f86e5634 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS presence(
   status_msg VARCHAR(150),
   mtime BIGINT, -- miliseconds since last state change
   UNIQUE (user_id)
-) ;
+);
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound(
   observed_user_id VARCHAR(150) NOT NULL,
   observer_user_id VARCHAR(150) NOT NULL, -- a UserID,
   UNIQUE (observed_user_id, observer_user_id)
-) ;
+);
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
@@ -35,4 +35,4 @@ CREATE TABLE IF NOT EXISTS presence_list(
   observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
   accepted BOOLEAN NOT NULL,
   UNIQUE (user_id, observed_user_id)
-) ;
+);
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 023060a54c..21c58a99bc 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -17,4 +17,4 @@ CREATE TABLE IF NOT EXISTS profiles(
     displayname VARCHAR(150),
     avatar_url VARCHAR(150),
     UNIQUE(user_id)
-) ;
+);
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 5c23188d62..b81451eab4 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS redactions (
     event_id VARCHAR(150) NOT NULL,
     redacts VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
 CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 63fe0f5c64..952cae35b7 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -17,9 +17,9 @@ CREATE TABLE IF NOT EXISTS room_aliases(
     room_alias VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     UNIQUE (room_alias)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
     room_alias VARCHAR(150) NOT NULL,
     server VARCHAR(150) NOT NULL
-) ;
+);
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index acfb76439b..737c3e35c7 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS state_groups(
     id VARCHAR(20) PRIMARY KEY,
     room_id VARCHAR(150) NOT NULL,
     event_id VARCHAR(150) NOT NULL
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group VARCHAR(20) NOT NULL,
@@ -25,13 +25,13 @@ CREATE TABLE IF NOT EXISTS state_groups_state(
     type VARCHAR(150) NOT NULL,
     state_key VARCHAR(150) NOT NULL,
     event_id VARCHAR(150) NOT NULL
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
     event_id VARCHAR(150) NOT NULL,
     state_group VARCHAR(150) NOT NULL,
     UNIQUE (event_id)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
 
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 43541661ce..c908109b6b 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS received_transactions(
     response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
     ts BIGINT
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
 CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
@@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     pdu_id VARCHAR(150),
     pdu_origin VARCHAR(150),
     UNIQUE (transaction_id, destination)
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
@@ -60,4 +60,4 @@ CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(150) PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
-) ;
+);
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index ba0f42d455..32dfc5b182 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS users(
     creation_ts BIGINT,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS access_tokens(
     id BIGINT PRIMARY KEY,
@@ -27,7 +27,7 @@ CREATE TABLE IF NOT EXISTS access_tokens(
     token VARCHAR(150) NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
-) ;
+);
 
 CREATE TABLE IF NOT EXISTS user_ips (
     user VARCHAR(150) NOT NULL,
@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
     last_seen BIGINT NOT NULL
-) ;
+);
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
 CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
-- 
cgit 1.5.1


From a1665c50949aaa257f3ee57fa02d1934b101642e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 10:05:44 +0100
Subject: Revert non-trivial schema changes and move them to a new schema
 version.

---
 synapse/storage/schema/full_schemas/11/im.sql      |  19 ++--
 .../schema/full_schemas/11/media_repository.sql    |   3 -
 .../storage/schema/full_schemas/11/presence.sql    |   9 +-
 .../storage/schema/full_schemas/11/profiles.sql    |   3 +-
 .../schema/full_schemas/11/room_aliases.sql        |   3 +-
 .../schema/full_schemas/11/transactions.sql        |   4 +-
 synapse/storage/schema/full_schemas/11/users.sql   |   5 +-
 .../storage/schema/full_schemas/16/event_edges.sql |  89 +++++++++++++++
 .../schema/full_schemas/16/event_signatures.sql    |  55 +++++++++
 synapse/storage/schema/full_schemas/16/im.sql      | 124 +++++++++++++++++++++
 synapse/storage/schema/full_schemas/16/keys.sql    |  31 ++++++
 .../schema/full_schemas/16/media_repository.sql    |  68 +++++++++++
 .../storage/schema/full_schemas/16/presence.sql    |  38 +++++++
 .../storage/schema/full_schemas/16/profiles.sql    |  20 ++++
 .../storage/schema/full_schemas/16/redactions.sql  |  22 ++++
 .../schema/full_schemas/16/room_aliases.sql        |  25 +++++
 synapse/storage/schema/full_schemas/16/state.sql   |  40 +++++++
 .../schema/full_schemas/16/transactions.sql        |  63 +++++++++++
 synapse/storage/schema/full_schemas/16/users.sql   |  42 +++++++
 19 files changed, 636 insertions(+), 27 deletions(-)
 create mode 100644 synapse/storage/schema/full_schemas/16/event_edges.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/event_signatures.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/im.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/keys.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/media_repository.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/presence.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/profiles.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/redactions.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/room_aliases.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/state.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/transactions.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/users.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 363e2c842e..eac9c6af4a 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering BIGINT PRIMARY KEY,
+    stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT,
     topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
@@ -62,10 +62,10 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     room_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     state_key VARCHAR(150) NOT NULL,
-    UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
 );
 
+CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
 CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
 CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
 CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
@@ -75,10 +75,10 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     user_id VARCHAR(150) NOT NULL,
     sender VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    membership VARCHAR(150) NOT NULL,
-    UNIQUE (event_id)
+    membership VARCHAR(150) NOT NULL
 );
 
+CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
 CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
 
@@ -87,26 +87,25 @@ CREATE TABLE IF NOT EXISTS feedback(
     feedback_type VARCHAR(150),
     target_event_id VARCHAR(150),
     sender VARCHAR(150),
-    room_id VARCHAR(150),
-    UNIQUE (event_id)
+    room_id VARCHAR(150)
 );
 
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    topic VARCHAR(150) NOT NULL,
-    UNIQUE (event_id)
+    topic VARCHAR(150) NOT NULL
 );
 
+CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
 CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    name VARCHAR(150) NOT NULL,
-    UNIQUE (event_id)
+    name VARCHAR(150) NOT NULL
 );
 
+CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
 CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 08c42722d2..c6ce6cf69f 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -63,6 +63,3 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
         thumbnail_type
      )
 );
-
-CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
-    ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index f3f86e5634..fce324b890 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -16,16 +16,14 @@ CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
   status_msg VARCHAR(150),
-  mtime BIGINT, -- miliseconds since last state change
-  UNIQUE (user_id)
+  mtime BIGINT -- miliseconds since last state change
 );
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
   observed_user_id VARCHAR(150) NOT NULL,
-  observer_user_id VARCHAR(150) NOT NULL, -- a UserID,
-  UNIQUE (observed_user_id, observer_user_id)
+  observer_user_id VARCHAR(150) NOT NULL -- a UserID,
 );
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
@@ -33,6 +31,5 @@ CREATE TABLE IF NOT EXISTS presence_allow_inbound(
 CREATE TABLE IF NOT EXISTS presence_list(
   user_id VARCHAR(150) NOT NULL,
   observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
-  accepted BOOLEAN NOT NULL,
-  UNIQUE (user_id, observed_user_id)
+  accepted BOOLEAN NOT NULL
 );
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index 21c58a99bc..ffe75edf9f 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -15,6 +15,5 @@
 CREATE TABLE IF NOT EXISTS profiles(
     user_id VARCHAR(150) NOT NULL,
     displayname VARCHAR(150),
-    avatar_url VARCHAR(150),
-    UNIQUE(user_id)
+    avatar_url VARCHAR(150)
 );
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 952cae35b7..6226913227 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -15,8 +15,7 @@
 
 CREATE TABLE IF NOT EXISTS room_aliases(
     room_alias VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    UNIQUE (room_alias)
+    room_id VARCHAR(150) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index c908109b6b..c3d46763a6 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -49,10 +49,10 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
     destination VARCHAR(150),
     pdu_id VARCHAR(150),
-    pdu_origin VARCHAR(150),
-    UNIQUE (transaction_id, destination)
+    pdu_origin VARCHAR(150)
 );
 
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination)
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 32dfc5b182..94c91e5298 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,6 +13,7 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
+    id BIGINT PRIMARY KEY AUTOINCREMENT,
     name VARCHAR(150),
     password_hash VARCHAR(150),
     creation_ts BIGINT,
@@ -35,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT NOT NULL
+    last_seen BIGINT NOT NULL,
+    UNIQUE (user, access_token, ip, user_agent)
 );
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
new file mode 100644
index 0000000000..602c11411d
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -0,0 +1,89 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS event_forward_extremities(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    UNIQUE (event_id, room_id)
+);
+
+CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
+CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_backward_extremities(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    UNIQUE (event_id, room_id)
+);
+
+CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
+CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_edges(
+    event_id VARCHAR(150) NOT NULL,
+    prev_event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    is_state BOOL NOT NULL,
+    UNIQUE (event_id, prev_event_id, room_id, is_state)
+);
+
+CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
+CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
+
+
+CREATE TABLE IF NOT EXISTS room_depth(
+    room_id VARCHAR(150) NOT NULL,
+    min_depth INTEGER NOT NULL,
+    UNIQUE (room_id)
+);
+
+CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
+
+
+create TABLE IF NOT EXISTS event_destinations(
+    event_id VARCHAR(150) NOT NULL,
+    destination VARCHAR(150) NOT NULL,
+    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
+    UNIQUE (event_id, destination)
+);
+
+CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
+
+
+CREATE TABLE IF NOT EXISTS state_forward_extremities(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    UNIQUE (event_id, room_id)
+);
+
+CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
+    room_id, type, state_key
+);
+CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_auth(
+    event_id VARCHAR(150) NOT NULL,
+    auth_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    UNIQUE (event_id, auth_id, room_id)
+);
+
+CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
+CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql
new file mode 100644
index 0000000000..0c27bc1f34
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql
@@ -0,0 +1,55 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS event_content_hashes (
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
+    hash BLOB,
+    UNIQUE (event_id, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_reference_hashes (
+    event_id VARCHAR(150),
+    algorithm VARCHAR(150),
+    hash BLOB,
+    UNIQUE (event_id, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_signatures (
+    event_id VARCHAR(150),
+    signature_name VARCHAR(150),
+    key_id VARCHAR(150),
+    signature BLOB,
+    UNIQUE (event_id, signature_name, key_id)
+);
+
+CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
+
+
+CREATE TABLE IF NOT EXISTS event_edge_hashes(
+    event_id VARCHAR(150),
+    prev_event_id VARCHAR(150),
+    algorithm VARCHAR(150),
+    hash BLOB,
+    UNIQUE (event_id, prev_event_id, algorithm)
+);
+
+CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
new file mode 100644
index 0000000000..363e2c842e
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -0,0 +1,124 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS events(
+    stream_ordering BIGINT PRIMARY KEY,
+    topological_ordering BIGINT NOT NULL,
+    event_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    content BLOB NOT NULL,
+    unrecognized_keys BLOB,
+    processed BOOL NOT NULL,
+    outlier BOOL NOT NULL,
+    depth BIGINT DEFAULT 0 NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
+CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
+CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
+
+
+CREATE TABLE IF NOT EXISTS event_json(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    internal_metadata BLOB NOT NULL,
+    json BLOB NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
+
+
+CREATE TABLE IF NOT EXISTS state_events(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    prev_state VARCHAR(150),
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
+CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
+CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
+
+
+CREATE TABLE IF NOT EXISTS current_state_events(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    UNIQUE (event_id),
+    UNIQUE (room_id, type, state_key)
+);
+
+CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
+CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
+CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
+
+CREATE TABLE IF NOT EXISTS room_memberships(
+    event_id VARCHAR(150) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
+    sender VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    membership VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
+CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
+
+CREATE TABLE IF NOT EXISTS feedback(
+    event_id VARCHAR(150) NOT NULL,
+    feedback_type VARCHAR(150),
+    target_event_id VARCHAR(150),
+    sender VARCHAR(150),
+    room_id VARCHAR(150),
+    UNIQUE (event_id)
+);
+
+CREATE TABLE IF NOT EXISTS topics(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    topic VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
+
+CREATE TABLE IF NOT EXISTS room_names(
+    event_id VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    name VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
+
+CREATE TABLE IF NOT EXISTS rooms(
+    room_id VARCHAR(150) PRIMARY KEY NOT NULL,
+    is_public BOOL,
+    creator VARCHAR(150)
+);
+
+CREATE TABLE IF NOT EXISTS room_hosts(
+    room_id VARCHAR(150) NOT NULL,
+    host VARCHAR(150) NOT NULL,
+    UNIQUE (room_id, host)
+);
+
+CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
new file mode 100644
index 0000000000..0e80bf761e
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -0,0 +1,31 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS server_tls_certificates(
+  server_name VARCHAR(150), -- Server name.
+  fingerprint VARCHAR(150), -- Certificate fingerprint.
+  from_server VARCHAR(150), -- Which key server the certificate was fetched from.
+  ts_added_ms BIGINT, -- When the certifcate was added.
+  tls_certificate BLOB, -- DER encoded x509 certificate.
+  UNIQUE (server_name, fingerprint)
+);
+
+CREATE TABLE IF NOT EXISTS server_signature_keys(
+  server_name VARCHAR(150), -- Server name.
+  key_id VARCHAR(150), -- Key version.
+  from_server VARCHAR(150), -- Which key server the key was fetched form.
+  ts_added_ms BIGINT, -- When the key was added.
+  verify_key BLOB, -- NACL verification key.
+  UNIQUE (server_name, key_id)
+);
diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql
new file mode 100644
index 0000000000..08c42722d2
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/media_repository.sql
@@ -0,0 +1,68 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS local_media_repository (
+    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_type VARCHAR(150), -- The MIME-type of the media.
+    media_length INTEGER, -- Length of the media in bytes.
+    created_ts BIGINT, -- When the content was uploaded in ms.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    user_id VARCHAR(150), -- The user who uploaded the file.
+    UNIQUE (media_id)
+);
+
+CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
+    media_id VARCHAR(150), -- The id used to refer to the media.
+    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail.
+    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+    UNIQUE (
+        media_id, thumbnail_width, thumbnail_height, thumbnail_type
+    )
+);
+
+CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
+    ON local_media_repository_thumbnails (media_id);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache (
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media on that server.
+    media_type VARCHAR(150), -- The MIME-type of the media.
+    created_ts BIGINT, -- When the content was uploaded in ms.
+    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    media_length INTEGER, -- Length of the media in bytes.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    UNIQUE (media_origin, media_id)
+);
+
+CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
+    media_origin VARCHAR(150), -- The remote HS the media came from.
+    media_id VARCHAR(150), -- The id used to refer to the media.
+    thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
+    thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
+    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail
+    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
+    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    UNIQUE (
+        media_origin, media_id, thumbnail_width, thumbnail_height,
+        thumbnail_type
+     )
+);
+
+CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
+    ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
new file mode 100644
index 0000000000..f3f86e5634
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -0,0 +1,38 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS presence(
+  user_id VARCHAR(150) NOT NULL,
+  state VARCHAR(20),
+  status_msg VARCHAR(150),
+  mtime BIGINT, -- miliseconds since last state change
+  UNIQUE (user_id)
+);
+
+-- For each of /my/ users which possibly-remote users are allowed to see their
+-- presence state
+CREATE TABLE IF NOT EXISTS presence_allow_inbound(
+  observed_user_id VARCHAR(150) NOT NULL,
+  observer_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  UNIQUE (observed_user_id, observer_user_id)
+);
+
+-- For each of /my/ users (watcher), which possibly-remote users are they
+-- watching?
+CREATE TABLE IF NOT EXISTS presence_list(
+  user_id VARCHAR(150) NOT NULL,
+  observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  accepted BOOLEAN NOT NULL,
+  UNIQUE (user_id, observed_user_id)
+);
diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql
new file mode 100644
index 0000000000..21c58a99bc
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/profiles.sql
@@ -0,0 +1,20 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS profiles(
+    user_id VARCHAR(150) NOT NULL,
+    displayname VARCHAR(150),
+    avatar_url VARCHAR(150),
+    UNIQUE(user_id)
+);
diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql
new file mode 100644
index 0000000000..b81451eab4
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/redactions.sql
@@ -0,0 +1,22 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS redactions (
+    event_id VARCHAR(150) NOT NULL,
+    redacts VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
+CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql
new file mode 100644
index 0000000000..952cae35b7
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql
@@ -0,0 +1,25 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS room_aliases(
+    room_alias VARCHAR(150) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    UNIQUE (room_alias)
+);
+
+CREATE TABLE IF NOT EXISTS room_alias_servers(
+    room_alias VARCHAR(150) NOT NULL,
+    server VARCHAR(150) NOT NULL
+);
diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
new file mode 100644
index 0000000000..737c3e35c7
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -0,0 +1,40 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS state_groups(
+    id VARCHAR(20) PRIMARY KEY,
+    room_id VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS state_groups_state(
+    state_group VARCHAR(20) NOT NULL,
+    room_id VARCHAR(150) NOT NULL,
+    type VARCHAR(150) NOT NULL,
+    state_key VARCHAR(150) NOT NULL,
+    event_id VARCHAR(150) NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS event_to_state_groups(
+    event_id VARCHAR(150) NOT NULL,
+    state_group VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
+
+CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group);
+CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
+CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
\ No newline at end of file
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
new file mode 100644
index 0000000000..c908109b6b
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -0,0 +1,63 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-- Stores what transaction ids we have received and what our response was
+CREATE TABLE IF NOT EXISTS received_transactions(
+    transaction_id VARCHAR(150),
+    origin VARCHAR(150),
+    ts BIGINT,
+    response_code INTEGER,
+    response_json BLOB,
+    has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
+    UNIQUE (transaction_id, origin)
+);
+
+CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
+
+
+-- Stores what transactions we've sent, what their response was (if we got one) and whether we have
+-- since referenced the transaction in another outgoing transaction
+CREATE TABLE IF NOT EXISTS sent_transactions(
+    id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
+    transaction_id VARCHAR(150),
+    destination VARCHAR(150),
+    response_code INTEGER DEFAULT 0,
+    response_json BLOB,
+    ts BIGINT
+);
+
+CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
+CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
+-- So that we can do an efficient look up of all transactions that have yet to be successfully
+-- sent.
+CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code);
+
+
+-- For sent transactions only.
+CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
+    transaction_id INTEGER,
+    destination VARCHAR(150),
+    pdu_id VARCHAR(150),
+    pdu_origin VARCHAR(150),
+    UNIQUE (transaction_id, destination)
+);
+
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
+
+-- To track destination health
+CREATE TABLE IF NOT EXISTS destinations(
+    destination VARCHAR(150) PRIMARY KEY,
+    retry_last_ts BIGINT,
+    retry_interval INTEGER
+);
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
new file mode 100644
index 0000000000..32dfc5b182
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -0,0 +1,42 @@
+/* Copyright 2014, 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+CREATE TABLE IF NOT EXISTS users(
+    name VARCHAR(150),
+    password_hash VARCHAR(150),
+    creation_ts BIGINT,
+    admin BOOL DEFAULT 0 NOT NULL,
+    UNIQUE(name)
+);
+
+CREATE TABLE IF NOT EXISTS access_tokens(
+    id BIGINT PRIMARY KEY,
+    user_id VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    token VARCHAR(150) NOT NULL,
+    last_used BIGINT,
+    UNIQUE(token)
+);
+
+CREATE TABLE IF NOT EXISTS user_ips (
+    user VARCHAR(150) NOT NULL,
+    access_token VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    ip VARCHAR(150) NOT NULL,
+    user_agent VARCHAR(150) NOT NULL,
+    last_seen BIGINT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
-- 
cgit 1.5.1


From e2722f58ee98e4af7b54dfa230f63520b1ba8558 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 10:16:29 +0100
Subject: Fix schema again

---
 synapse/storage/schema/full_schemas/11/transactions.sql | 4 ++--
 synapse/storage/schema/full_schemas/11/users.sql        | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index c3d46763a6..093bb275b4 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
+    id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
@@ -52,7 +52,7 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     pdu_origin VARCHAR(150)
 );
 
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination)
+CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
 CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 94c91e5298..e9a9fc94f7 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users(
 );
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id BIGINT PRIMARY KEY,
+    id BIGINT PRIMARY KEY AUTOINCREMENT,
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-- 
cgit 1.5.1


From cda4a6f93f695cf35eea334ba17e8694aeeab58c Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 10:19:50 +0100
Subject: Revert non-trivial changes to upgrade scripts

---
 synapse/storage/schema/delta/12/v12.sql | 4 ++--
 synapse/storage/schema/delta/13/v13.sql | 4 ++--
 synapse/storage/schema/delta/14/v14.sql | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index ed8a2f3f10..f0d7e74b8e 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections(
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id BIGINT PRIMARY KEY,
+  id BIGINT PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
@@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers (
 );
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id BIGINT PRIMARY KEY,
+  id BIGINT PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   priority_class TINYINT NOT NULL,
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index ef6551f3ad..202adc7200 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id BIGINT PRIMARY KEY,
+    id BIGINT PRIMARY KEY AUTOINCREMENT,
     url VARCHAR(150),
     token VARCHAR(150),
     hs_token VARCHAR(150),
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services(
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id BIGINT PRIMARY KEY,
+    id BIGINT PRIMARY KEY AUTOINCREMENT,
     as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index f1208c731d..db091fcd5e 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id BIGINT PRIMARY KEY,
+  id BIGINT PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
-- 
cgit 1.5.1


From d5d4281647246f14630bebba1820057db7d81c87 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 10:59:46 +0100
Subject: Update full_schemas/16 to match delta files. Add delta/16 scripts

---
 .../schema/delta/16/remote_media_cache_index.sql   |  2 +
 .../storage/schema/delta/16/unique_constraints.sql | 64 +++++++++++++++++++
 synapse/storage/schema/delta/16/users.sql          | 56 +++++++++++++++++
 .../full_schemas/16/application_services.sql       | 48 ++++++++++++++
 .../storage/schema/full_schemas/16/presence.sql    |  2 +
 synapse/storage/schema/full_schemas/16/push.sql    | 73 ++++++++++++++++++++++
 6 files changed, 245 insertions(+)
 create mode 100644 synapse/storage/schema/delta/16/remote_media_cache_index.sql
 create mode 100644 synapse/storage/schema/delta/16/unique_constraints.sql
 create mode 100644 synapse/storage/schema/delta/16/users.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/application_services.sql
 create mode 100644 synapse/storage/schema/full_schemas/16/push.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/delta/16/remote_media_cache_index.sql
new file mode 100644
index 0000000000..7a15265cb1
--- /dev/null
+++ b/synapse/storage/schema/delta/16/remote_media_cache_index.sql
@@ -0,0 +1,2 @@
+CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
+    ON remote_media_cache_thumbnails (media_id);
\ No newline at end of file
diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql
new file mode 100644
index 0000000000..b2b677e510
--- /dev/null
+++ b/synapse/storage/schema/delta/16/unique_constraints.sql
@@ -0,0 +1,64 @@
+
+-- We can use SQLite features here, since mysql support was only added in v16
+
+--
+DELETE FROM current_state_events WHERE rowid not in (
+    SELECT MIN(rowid) FROM current_state_events GROUP BY event_id
+);
+
+CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id);
+
+--
+DELETE FROM room_memberships WHERE rowid not in (
+    SELECT MIN(rowid) FROM room_memberships GROUP BY event_id
+);
+
+CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id);
+
+--
+DELETE FROM feedback WHERE rowid not in (
+    SELECT MIN(rowid) FROM feedback GROUP BY event_id
+);
+
+CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id);
+
+--
+DELETE FROM topics WHERE rowid not in (
+    SELECT MIN(rowid) FROM topics GROUP BY event_id
+);
+
+CREATE UNIQUE INDEX topics_event_id ON topics(event_id);
+
+--
+DELETE FROM room_names WHERE rowid not in (
+    SELECT MIN(rowid) FROM room_names GROUP BY event_id
+);
+
+CREATE UNIQUE INDEX room_names_id ON room_names(event_id);
+
+--
+DELETE FROM presence WHERE rowid not in (
+    SELECT MIN(rowid) FROM presence GROUP BY user_id
+);
+
+CREATE UNIQUE INDEX presence_id ON presence(user_id);
+
+--
+DELETE FROM presence_allow_inbound WHERE rowid not in (
+    SELECT MIN(rowid) FROM presence_allow_inbound
+    GROUP BY observed_user_id, observer_user_id
+);
+
+CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound(
+    observed_user_id, observer_user_id
+);
+
+--
+DELETE FROM presence_list WHERE rowid not in (
+    SELECT MIN(rowid) FROM presence_list
+    GROUP BY user_id, observed_user_id
+);
+
+CREATE UNIQUE INDEX presence_list_observers ON presence_list(
+    user_id, observed_user_id
+);
diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql
new file mode 100644
index 0000000000..1bbd1c0a17
--- /dev/null
+++ b/synapse/storage/schema/delta/16/users.sql
@@ -0,0 +1,56 @@
+-- Convert `access_tokens`.user from rowids to user strings.
+-- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW
+CREATE TABLE IF NOT EXISTS new_access_tokens(
+    id BIGINT PRIMARY KEY,
+    user_id VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    token VARCHAR(150) NOT NULL,
+    last_used BIGINT,
+    UNIQUE(token)
+);
+
+INSERT INTO new_access_tokens
+    SELECT a.id, u.name, a.device_id, a.token, a.last_used
+    FROM access_tokens as a
+    INNER JOIN users as u ON u.id = a.user_id;
+
+DROP TABLE access_tokens;
+
+ALTER TABLE new_access_tokens RENAME TO access_tokens;
+
+-- Remove ID column from `users` table
+CREATE TABLE IF NOT EXISTS new_users(
+    name VARCHAR(150),
+    password_hash VARCHAR(150),
+    creation_ts BIGINT,
+    admin BOOL DEFAULT 0 NOT NULL,
+    UNIQUE(name)
+);
+
+INSERT INTO new_users SELECT name, password_hash, creation_ts, admin FROM users;
+
+DROP TABLE users;
+
+ALTER TABLE new_users RENAME TO users;
+
+
+-- Remove UNIQUE constraint from `user_ips` table
+CREATE TABLE IF NOT EXISTS new_user_ips (
+    user VARCHAR(150) NOT NULL,
+    access_token VARCHAR(150) NOT NULL,
+    device_id VARCHAR(150),
+    ip VARCHAR(150) NOT NULL,
+    user_agent VARCHAR(150) NOT NULL,
+    last_seen BIGINT NOT NULL
+);
+
+INSERT INTO new_user_ips
+    SELECT user, access_token, device_id, ip, user_agent, last_seen FROM user_ips;
+
+DROP TABLE user_ips;
+
+ALTER TABLE new_user_ips RENAME TO user_ips;
+
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
+
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
new file mode 100644
index 0000000000..7a5c2510d2
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -0,0 +1,48 @@
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS application_services(
+    id BIGINT PRIMARY KEY,
+    url VARCHAR(150),
+    token VARCHAR(150),
+    hs_token VARCHAR(150),
+    sender VARCHAR(150),
+    UNIQUE(token)
+);
+
+CREATE TABLE IF NOT EXISTS application_services_regex(
+    id BIGINT PRIMARY KEY,
+    as_id BIGINT NOT NULL,
+    namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
+    regex VARCHAR(150),
+    FOREIGN KEY(as_id) REFERENCES application_services(id)
+);
+
+CREATE TABLE IF NOT EXISTS application_services_state(
+    as_id VARCHAR(150) PRIMARY KEY,
+    state VARCHAR(5),
+    last_txn INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS application_services_txns(
+    as_id VARCHAR(150) NOT NULL,
+    txn_id INTEGER NOT NULL,
+    event_ids BLOB NOT NULL,
+    UNIQUE(as_id, txn_id)
+);
+
+CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns (
+    as_id
+);
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
index f3f86e5634..15a6ad3c79 100644
--- a/synapse/storage/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -36,3 +36,5 @@ CREATE TABLE IF NOT EXISTS presence_list(
   accepted BOOLEAN NOT NULL,
   UNIQUE (user_id, observed_user_id)
 );
+
+CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id);
diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql
new file mode 100644
index 0000000000..4c8faa9213
--- /dev/null
+++ b/synapse/storage/schema/full_schemas/16/push.sql
@@ -0,0 +1,73 @@
+/* Copyright 2015 OpenMarket Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+CREATE TABLE IF NOT EXISTS rejections(
+    event_id VARCHAR(150) NOT NULL,
+    reason VARCHAR(150) NOT NULL,
+    last_check VARCHAR(150) NOT NULL,
+    UNIQUE (event_id)
+);
+
+-- Push notification endpoints that users have configured
+CREATE TABLE IF NOT EXISTS pushers (
+  id BIGINT PRIMARY KEY,
+  user_name VARCHAR(150) NOT NULL,
+  profile_tag VARCHAR(32) NOT NULL,
+  kind VARCHAR(8) NOT NULL,
+  app_id VARCHAR(64) NOT NULL,
+  app_display_name VARCHAR(64) NOT NULL,
+  device_display_name VARCHAR(128) NOT NULL,
+  pushkey VARBINARY(512) NOT NULL,
+  ts BIGINT NOT NULL,
+  lang VARCHAR(8),
+  data BLOB,
+  last_token TEXT,
+  last_success BIGINT,
+  failing_since BIGINT,
+  UNIQUE (app_id, pushkey)
+);
+
+CREATE TABLE IF NOT EXISTS push_rules (
+  id BIGINT PRIMARY KEY,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
+  priority_class TINYINT NOT NULL,
+  priority INTEGER NOT NULL DEFAULT 0,
+  conditions VARCHAR(150) NOT NULL,
+  actions VARCHAR(150) NOT NULL,
+  UNIQUE(user_name, rule_id)
+);
+
+CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
+
+CREATE TABLE IF NOT EXISTS user_filters(
+  user_id VARCHAR(150),
+  filter_id BIGINT,
+  filter_json BLOB
+);
+
+CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
+    user_id, filter_id
+);
+
+CREATE TABLE IF NOT EXISTS push_rules_enable (
+  id BIGINT PRIMARY KEY,
+  user_name VARCHAR(150) NOT NULL,
+  rule_id VARCHAR(150) NOT NULL,
+  enabled TINYINT,
+  UNIQUE(user_name, rule_id)
+);
+
+CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);
-- 
cgit 1.5.1


From 0f12772e320291408c8c7b475a391556db4c6d2b Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 11:16:09 +0100
Subject: SQLite wants INTEGER and not BIGINT for primary keys

---
 synapse/storage/schema/delta/12/v12.sql                 | 4 ++--
 synapse/storage/schema/delta/13/v13.sql                 | 4 ++--
 synapse/storage/schema/delta/14/v14.sql                 | 2 +-
 synapse/storage/schema/full_schemas/11/im.sql           | 2 +-
 synapse/storage/schema/full_schemas/11/transactions.sql | 2 +-
 synapse/storage/schema/full_schemas/11/users.sql        | 4 ++--
 6 files changed, 9 insertions(+), 9 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index f0d7e74b8e..4107cba71f 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections(
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id BIGINT PRIMARY KEY AUTOINCREMENT,
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
@@ -40,7 +40,7 @@ CREATE TABLE IF NOT EXISTS pushers (
 );
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id BIGINT PRIMARY KEY AUTOINCREMENT,
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   priority_class TINYINT NOT NULL,
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 202adc7200..17a139b867 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id BIGINT PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     url VARCHAR(150),
     token VARCHAR(150),
     hs_token VARCHAR(150),
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS application_services(
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id BIGINT PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index db091fcd5e..8c47d4b0f4 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -1,5 +1,5 @@
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id BIGINT PRIMARY KEY AUTOINCREMENT,
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index eac9c6af4a..91ae9b9ee4 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering BIGINT PRIMARY KEY AUTOINCREMENT,
+    stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
     topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 093bb275b4..a9bf301efc 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -29,7 +29,7 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id BIGINT PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
+    id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index e9a9fc94f7..ad5e403f92 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    id BIGINT PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     name VARCHAR(150),
     password_hash VARCHAR(150),
     creation_ts BIGINT,
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS users(
 );
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id BIGINT PRIMARY KEY AUTOINCREMENT,
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-- 
cgit 1.5.1


From 7e863c51e6a6a5b6f757cb279f5d7308db1432d4 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 11:22:04 +0100
Subject: Use unsigned bigint

---
 synapse/storage/schema/delta/12/v12.sql                    |  8 ++++----
 synapse/storage/schema/delta/13/v13.sql                    |  2 +-
 synapse/storage/schema/delta/16/users.sql                  |  8 ++++----
 synapse/storage/schema/full_schemas/11/event_edges.sql     |  2 +-
 synapse/storage/schema/full_schemas/11/im.sql              |  4 ++--
 synapse/storage/schema/full_schemas/11/keys.sql            |  4 ++--
 .../storage/schema/full_schemas/11/media_repository.sql    |  4 ++--
 synapse/storage/schema/full_schemas/11/presence.sql        |  2 +-
 synapse/storage/schema/full_schemas/11/transactions.sql    |  6 +++---
 synapse/storage/schema/full_schemas/11/users.sql           |  6 +++---
 .../schema/full_schemas/16/application_services.sql        |  6 +++---
 synapse/storage/schema/full_schemas/16/event_edges.sql     |  2 +-
 synapse/storage/schema/full_schemas/16/im.sql              |  6 +++---
 synapse/storage/schema/full_schemas/16/keys.sql            |  4 ++--
 .../storage/schema/full_schemas/16/media_repository.sql    |  4 ++--
 synapse/storage/schema/full_schemas/16/presence.sql        |  2 +-
 synapse/storage/schema/full_schemas/16/push.sql            | 14 +++++++-------
 synapse/storage/schema/full_schemas/16/transactions.sql    |  8 ++++----
 synapse/storage/schema/full_schemas/16/users.sql           |  8 ++++----
 19 files changed, 50 insertions(+), 50 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 4107cba71f..1247fd175c 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -30,12 +30,12 @@ CREATE TABLE IF NOT EXISTS pushers (
   app_display_name VARCHAR(64) NOT NULL,
   device_display_name VARCHAR(128) NOT NULL,
   pushkey VARBINARY(512) NOT NULL,
-  ts BIGINT NOT NULL,
+  ts BIGINT UNSIGNED NOT NULL,
   lang VARCHAR(8),
   data BLOB,
   last_token TEXT,
-  last_success BIGINT,
-  failing_since BIGINT,
+  last_success BIGINT UNSIGNED,
+  failing_since BIGINT UNSIGNED,
   UNIQUE (app_id, pushkey)
 );
 
@@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
-  filter_id BIGINT,
+  filter_id BIGINT UNSIGNED,
   filter_json BLOB
 );
 
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index 17a139b867..d1da2b48e2 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -24,7 +24,7 @@ CREATE TABLE IF NOT EXISTS application_services(
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    as_id BIGINT NOT NULL,
+    as_id BIGINT UNSIGNED NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql
index 1bbd1c0a17..f04705cd77 100644
--- a/synapse/storage/schema/delta/16/users.sql
+++ b/synapse/storage/schema/delta/16/users.sql
@@ -1,11 +1,11 @@
 -- Convert `access_tokens`.user from rowids to user strings.
 -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW
 CREATE TABLE IF NOT EXISTS new_access_tokens(
-    id BIGINT PRIMARY KEY,
+    id BIGINT UNSIGNED PRIMARY KEY,
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-    last_used BIGINT,
+    last_used BIGINT UNSIGNED,
     UNIQUE(token)
 );
 
@@ -22,7 +22,7 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens;
 CREATE TABLE IF NOT EXISTS new_users(
     name VARCHAR(150),
     password_hash VARCHAR(150),
-    creation_ts BIGINT,
+    creation_ts BIGINT UNSIGNED,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
 );
@@ -41,7 +41,7 @@ CREATE TABLE IF NOT EXISTS new_user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT NOT NULL
+    last_seen BIGINT UNSIGNED NOT NULL
 );
 
 INSERT INTO new_user_ips
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 602c11411d..bdb1109094 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(150) NOT NULL,
     destination VARCHAR(150) NOT NULL,
-    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 91ae9b9ee4..e75093a446 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -15,7 +15,7 @@
 
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
-    topological_ordering BIGINT NOT NULL,
+    topological_ordering BIGINT UNSIGNED NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events(
     unrecognized_keys BLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
-    depth BIGINT DEFAULT 0 NOT NULL,
+    depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 0e80bf761e..551650cf1d 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name VARCHAR(150), -- Server name.
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
-  ts_added_ms BIGINT, -- When the certifcate was added.
+  ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(150), -- Server name.
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
-  ts_added_ms BIGINT, -- When the key was added.
+  ts_added_ms BIGINT UNSIGNED, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index c6ce6cf69f..134e7fbcec 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     media_id VARCHAR(150), -- The id used to refer to the media.
     media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
-    created_ts BIGINT, -- When the content was uploaded in ms.
+    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_origin VARCHAR(150), -- The remote HS the media came from.
     media_id VARCHAR(150), -- The id used to refer to the media on that server.
     media_type VARCHAR(150), -- The MIME-type of the media.
-    created_ts BIGINT, -- When the content was uploaded in ms.
+    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(150), -- The name used to store the media on disk.
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index fce324b890..c617ebea73 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
   status_msg VARCHAR(150),
-  mtime BIGINT -- miliseconds since last state change
+  mtime BIGINT UNSIGNED -- miliseconds since last state change
 );
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index a9bf301efc..67c79db3e0 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -16,7 +16,7 @@
 CREATE TABLE IF NOT EXISTS received_transactions(
     transaction_id VARCHAR(150),
     origin VARCHAR(150),
-    ts BIGINT,
+    ts BIGINT UNSIGNED,
     response_code INTEGER,
     response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
@@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
-    ts BIGINT
+    ts BIGINT UNSIGNED
 );
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
@@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(150) PRIMARY KEY,
-    retry_last_ts BIGINT,
+    retry_last_ts BIGINT UNSIGNED,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index ad5e403f92..0ddfccd410 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS users(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     name VARCHAR(150),
     password_hash VARCHAR(150),
-    creation_ts BIGINT,
+    creation_ts BIGINT UNSIGNED,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
 );
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens(
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-    last_used BIGINT,
+    last_used BIGINT UNSIGNED,
     UNIQUE(token)
 );
 
@@ -36,7 +36,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT NOT NULL,
+    last_seen BIGINT UNSIGNED NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index 7a5c2510d2..fbc3e08a1c 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id BIGINT PRIMARY KEY,
+    id BIGINT UNSIGNED PRIMARY KEY,
     url VARCHAR(150),
     token VARCHAR(150),
     hs_token VARCHAR(150),
@@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services(
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id BIGINT PRIMARY KEY,
-    as_id BIGINT NOT NULL,
+    id BIGINT UNSIGNED PRIMARY KEY,
+    as_id BIGINT UNSIGNED NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
index 602c11411d..bdb1109094 100644
--- a/synapse/storage/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -57,7 +57,7 @@ CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(150) NOT NULL,
     destination VARCHAR(150) NOT NULL,
-    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 363e2c842e..81dae2a911 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering BIGINT PRIMARY KEY,
-    topological_ordering BIGINT NOT NULL,
+    stream_ordering BIGINT UNSIGNED PRIMARY KEY,
+    topological_ordering BIGINT UNSIGNED NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS events(
     unrecognized_keys BLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
-    depth BIGINT DEFAULT 0 NOT NULL,
+    depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index 0e80bf761e..551650cf1d 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name VARCHAR(150), -- Server name.
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
-  ts_added_ms BIGINT, -- When the certifcate was added.
+  ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
   tls_certificate BLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(150), -- Server name.
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
-  ts_added_ms BIGINT, -- When the key was added.
+  ts_added_ms BIGINT UNSIGNED, -- When the key was added.
   verify_key BLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql
index 08c42722d2..014bce4aeb 100644
--- a/synapse/storage/schema/full_schemas/16/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/16/media_repository.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     media_id VARCHAR(150), -- The id used to refer to the media.
     media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
-    created_ts BIGINT, -- When the content was uploaded in ms.
+    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_origin VARCHAR(150), -- The remote HS the media came from.
     media_id VARCHAR(150), -- The id used to refer to the media on that server.
     media_type VARCHAR(150), -- The MIME-type of the media.
-    created_ts BIGINT, -- When the content was uploaded in ms.
+    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(150), -- The name used to store the media on disk.
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
index 15a6ad3c79..fbe5b0af6c 100644
--- a/synapse/storage/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
   status_msg VARCHAR(150),
-  mtime BIGINT, -- miliseconds since last state change
+  mtime BIGINT UNSIGNED, -- miliseconds since last state change
   UNIQUE (user_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql
index 4c8faa9213..99175b9632 100644
--- a/synapse/storage/schema/full_schemas/16/push.sql
+++ b/synapse/storage/schema/full_schemas/16/push.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS rejections(
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id BIGINT PRIMARY KEY,
+  id BIGINT UNSIGNED PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
@@ -30,17 +30,17 @@ CREATE TABLE IF NOT EXISTS pushers (
   app_display_name VARCHAR(64) NOT NULL,
   device_display_name VARCHAR(128) NOT NULL,
   pushkey VARBINARY(512) NOT NULL,
-  ts BIGINT NOT NULL,
+  ts BIGINT UNSIGNED NOT NULL,
   lang VARCHAR(8),
   data BLOB,
   last_token TEXT,
-  last_success BIGINT,
-  failing_since BIGINT,
+  last_success BIGINT UNSIGNED,
+  failing_since BIGINT UNSIGNED,
   UNIQUE (app_id, pushkey)
 );
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id BIGINT PRIMARY KEY,
+  id BIGINT UNSIGNED PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   priority_class TINYINT NOT NULL,
@@ -54,7 +54,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
-  filter_id BIGINT,
+  filter_id BIGINT UNSIGNED,
   filter_json BLOB
 );
 
@@ -63,7 +63,7 @@ CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
 );
 
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id BIGINT PRIMARY KEY,
+  id BIGINT UNSIGNED PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
   enabled TINYINT,
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index c908109b6b..e48eb44144 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -16,7 +16,7 @@
 CREATE TABLE IF NOT EXISTS received_transactions(
     transaction_id VARCHAR(150),
     origin VARCHAR(150),
-    ts BIGINT,
+    ts BIGINT UNSIGNED,
     response_code INTEGER,
     response_json BLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
@@ -29,12 +29,12 @@ CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
+    id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
     response_json BLOB,
-    ts BIGINT
+    ts BIGINT UNSIGNED
 );
 
 CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
@@ -58,6 +58,6 @@ CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(d
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(150) PRIMARY KEY,
-    retry_last_ts BIGINT,
+    retry_last_ts BIGINT UNSIGNED,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
index 32dfc5b182..d0011c04b4 100644
--- a/synapse/storage/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -15,17 +15,17 @@
 CREATE TABLE IF NOT EXISTS users(
     name VARCHAR(150),
     password_hash VARCHAR(150),
-    creation_ts BIGINT,
+    creation_ts BIGINT UNSIGNED,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
 );
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id BIGINT PRIMARY KEY,
+    id BIGINT UNSIGNED PRIMARY KEY,
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-    last_used BIGINT,
+    last_used BIGINT UNSIGNED,
     UNIQUE(token)
 );
 
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT NOT NULL
+    last_seen BIGINT UNSIGNED NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-- 
cgit 1.5.1


From 4d5b0986261db7e9a4389c6fe2eb4808e5946e91 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 18:45:09 +0100
Subject: Use LONGBLOB and TEXT for arbitary length rows

---
 synapse/storage/schema/delta/12/v12.sql                      |  4 ++--
 synapse/storage/schema/delta/15/appservice_txns.sql          |  2 +-
 synapse/storage/schema/full_schemas/11/event_signatures.sql  |  8 ++++----
 synapse/storage/schema/full_schemas/11/im.sql                | 12 ++++++------
 synapse/storage/schema/full_schemas/11/keys.sql              |  4 ++--
 synapse/storage/schema/full_schemas/11/transactions.sql      |  4 ++--
 .../storage/schema/full_schemas/16/application_services.sql  |  2 +-
 synapse/storage/schema/full_schemas/16/event_signatures.sql  |  8 ++++----
 synapse/storage/schema/full_schemas/16/im.sql                | 12 ++++++------
 synapse/storage/schema/full_schemas/16/keys.sql              |  4 ++--
 synapse/storage/schema/full_schemas/16/push.sql              |  4 ++--
 synapse/storage/schema/full_schemas/16/transactions.sql      |  4 ++--
 12 files changed, 34 insertions(+), 34 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index 1247fd175c..a246943f5a 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers (
   pushkey VARBINARY(512) NOT NULL,
   ts BIGINT UNSIGNED NOT NULL,
   lang VARCHAR(8),
-  data BLOB,
+  data LONGBLOB,
   last_token TEXT,
   last_success BIGINT UNSIGNED,
   failing_since BIGINT UNSIGNED,
@@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
   filter_id BIGINT UNSIGNED,
-  filter_json BLOB
+  filter_json LONGBLOB
 );
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql
index 2f4c3eae5f..ddea8fc693 100644
--- a/synapse/storage/schema/delta/15/appservice_txns.sql
+++ b/synapse/storage/schema/delta/15/appservice_txns.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids BLOB NOT NULL,
+    event_ids LONGBLOB NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 0c27bc1f34..09886f607c 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -16,7 +16,7 @@
 CREATE TABLE IF NOT EXISTS event_content_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, algorithm)
 );
 
@@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, algorithm)
 );
 
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     event_id VARCHAR(150),
     signature_name VARCHAR(150),
     key_id VARCHAR(150),
-    signature BLOB,
+    signature LONGBLOB,
     UNIQUE (event_id, signature_name, key_id)
 );
 
@@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes(
     event_id VARCHAR(150),
     prev_event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index e75093a446..9c47f51742 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content BLOB NOT NULL,
-    unrecognized_keys BLOB,
+    content LONGBLOB NOT NULL,
+    unrecognized_keys LONGBLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
@@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata BLOB NOT NULL,
-    json BLOB NOT NULL,
+    internal_metadata LONGBLOB NOT NULL,
+    json LONGBLOB NOT NULL,
     UNIQUE (event_id)
 );
 
@@ -93,7 +93,7 @@ CREATE TABLE IF NOT EXISTS feedback(
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    topic VARCHAR(150) NOT NULL
+    topic TEXT NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
@@ -102,7 +102,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 CREATE TABLE IF NOT EXISTS room_names(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    name VARCHAR(150) NOT NULL
+    name TEXT NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 551650cf1d..35f141c288 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
-  tls_certificate BLOB, -- DER encoded x509 certificate.
+  tls_certificate LONGBLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
@@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
   ts_added_ms BIGINT UNSIGNED, -- When the key was added.
-  verify_key BLOB, -- NACL verification key.
+  verify_key LONGBLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 67c79db3e0..c2fab10aa0 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions(
     origin VARCHAR(150),
     ts BIGINT UNSIGNED,
     response_code INTEGER,
-    response_json BLOB,
+    response_json LONGBLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
 );
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json BLOB,
+    response_json LONGBLOB,
     ts BIGINT UNSIGNED
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index fbc3e08a1c..bc709df92d 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids BLOB NOT NULL,
+    event_ids LONGBLOB NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql
index 0c27bc1f34..09886f607c 100644
--- a/synapse/storage/schema/full_schemas/16/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql
@@ -16,7 +16,7 @@
 CREATE TABLE IF NOT EXISTS event_content_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, algorithm)
 );
 
@@ -26,7 +26,7 @@ CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, algorithm)
 );
 
@@ -37,7 +37,7 @@ CREATE TABLE IF NOT EXISTS event_signatures (
     event_id VARCHAR(150),
     signature_name VARCHAR(150),
     key_id VARCHAR(150),
-    signature BLOB,
+    signature LONGBLOB,
     UNIQUE (event_id, signature_name, key_id)
 );
 
@@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS event_edge_hashes(
     event_id VARCHAR(150),
     prev_event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash BLOB,
+    hash LONGBLOB,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 81dae2a911..19f0f34143 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content BLOB NOT NULL,
-    unrecognized_keys BLOB,
+    content LONGBLOB NOT NULL,
+    unrecognized_keys LONGBLOB,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
@@ -35,8 +35,8 @@ CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata BLOB NOT NULL,
-    json BLOB NOT NULL,
+    internal_metadata LONGBLOB NOT NULL,
+    json LONGBLOB NOT NULL,
     UNIQUE (event_id)
 );
 
@@ -94,7 +94,7 @@ CREATE TABLE IF NOT EXISTS feedback(
 CREATE TABLE IF NOT EXISTS topics(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    topic VARCHAR(150) NOT NULL,
+    topic TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
@@ -103,7 +103,7 @@ CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
 CREATE TABLE IF NOT EXISTS room_names(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    name VARCHAR(150) NOT NULL,
+    name TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index 551650cf1d..35f141c288 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
-  tls_certificate BLOB, -- DER encoded x509 certificate.
+  tls_certificate LONGBLOB, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
@@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
   ts_added_ms BIGINT UNSIGNED, -- When the key was added.
-  verify_key BLOB, -- NACL verification key.
+  verify_key LONGBLOB, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql
index 99175b9632..33300736f9 100644
--- a/synapse/storage/schema/full_schemas/16/push.sql
+++ b/synapse/storage/schema/full_schemas/16/push.sql
@@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS pushers (
   pushkey VARBINARY(512) NOT NULL,
   ts BIGINT UNSIGNED NOT NULL,
   lang VARCHAR(8),
-  data BLOB,
+  data LONGBLOB,
   last_token TEXT,
   last_success BIGINT UNSIGNED,
   failing_since BIGINT UNSIGNED,
@@ -55,7 +55,7 @@ CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
   filter_id BIGINT UNSIGNED,
-  filter_json BLOB
+  filter_json LONGBLOB
 );
 
 CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index e48eb44144..f381e67603 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS received_transactions(
     origin VARCHAR(150),
     ts BIGINT UNSIGNED,
     response_code INTEGER,
-    response_json BLOB,
+    response_json LONGBLOB,
     has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
 );
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json BLOB,
+    response_json LONGBLOB,
     ts BIGINT UNSIGNED
 );
 
-- 
cgit 1.5.1


From 4103b1c4709b374785ffb7eca75c8f338218bc17 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 18:46:09 +0100
Subject: DROP indexes before recreating them

---
 synapse/storage/schema/delta/16/unique_constraints.sql | 8 ++++++++
 synapse/storage/schema/full_schemas/16/state.sql       | 2 +-
 2 files changed, 9 insertions(+), 1 deletion(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql
index b2b677e510..f9fbb6b448 100644
--- a/synapse/storage/schema/delta/16/unique_constraints.sql
+++ b/synapse/storage/schema/delta/16/unique_constraints.sql
@@ -6,6 +6,7 @@ DELETE FROM current_state_events WHERE rowid not in (
     SELECT MIN(rowid) FROM current_state_events GROUP BY event_id
 );
 
+DROP INDEX IF EXISTS current_state_events_event_id;
 CREATE UNIQUE INDEX current_state_events_event_id ON current_state_events(event_id);
 
 --
@@ -13,6 +14,7 @@ DELETE FROM room_memberships WHERE rowid not in (
     SELECT MIN(rowid) FROM room_memberships GROUP BY event_id
 );
 
+DROP INDEX IF EXISTS room_memberships_event_id;
 CREATE UNIQUE INDEX room_memberships_event_id ON room_memberships(event_id);
 
 --
@@ -20,6 +22,7 @@ DELETE FROM feedback WHERE rowid not in (
     SELECT MIN(rowid) FROM feedback GROUP BY event_id
 );
 
+DROP INDEX IF EXISTS feedback_event_id;
 CREATE UNIQUE INDEX feedback_event_id ON feedback(event_id);
 
 --
@@ -27,6 +30,7 @@ DELETE FROM topics WHERE rowid not in (
     SELECT MIN(rowid) FROM topics GROUP BY event_id
 );
 
+DROP INDEX IF EXISTS topics_event_id;
 CREATE UNIQUE INDEX topics_event_id ON topics(event_id);
 
 --
@@ -34,6 +38,7 @@ DELETE FROM room_names WHERE rowid not in (
     SELECT MIN(rowid) FROM room_names GROUP BY event_id
 );
 
+DROP INDEX IF EXISTS room_names_id;
 CREATE UNIQUE INDEX room_names_id ON room_names(event_id);
 
 --
@@ -41,6 +46,7 @@ DELETE FROM presence WHERE rowid not in (
     SELECT MIN(rowid) FROM presence GROUP BY user_id
 );
 
+DROP INDEX IF EXISTS presence_id;
 CREATE UNIQUE INDEX presence_id ON presence(user_id);
 
 --
@@ -49,6 +55,7 @@ DELETE FROM presence_allow_inbound WHERE rowid not in (
     GROUP BY observed_user_id, observer_user_id
 );
 
+DROP INDEX IF EXISTS presence_allow_inbound_observers;
 CREATE UNIQUE INDEX presence_allow_inbound_observers ON presence_allow_inbound(
     observed_user_id, observer_user_id
 );
@@ -59,6 +66,7 @@ DELETE FROM presence_list WHERE rowid not in (
     GROUP BY user_id, observed_user_id
 );
 
+DROP INDEX IF EXISTS presence_list_observers;
 CREATE UNIQUE INDEX presence_list_observers ON presence_list(
     user_id, observed_user_id
 );
diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
index 737c3e35c7..8d6b94a616 100644
--- a/synapse/storage/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -37,4 +37,4 @@ CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
 
 CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group);
 CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
-CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
\ No newline at end of file
+CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
-- 
cgit 1.5.1


From 93937b2b313e5b1020d21f3f2cad404ecff659c2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 10 Apr 2015 18:47:03 +0100
Subject: Remove duplicate rows

---
 synapse/storage/schema/delta/16/remove_duplicates.sql | 9 +++++++++
 1 file changed, 9 insertions(+)
 create mode 100644 synapse/storage/schema/delta/16/remove_duplicates.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/delta/16/remove_duplicates.sql
new file mode 100644
index 0000000000..65c97b5e2f
--- /dev/null
+++ b/synapse/storage/schema/delta/16/remove_duplicates.sql
@@ -0,0 +1,9 @@
+
+
+DELETE FROM event_to_state_groups WHERE state_group not in (
+    SELECT MAX(state_group) FROM event_to_state_groups GROUP BY event_id
+);
+
+DELETE FROM event_to_state_groups WHERE rowid not in (
+    SELECT MIN(rowid) FROM event_to_state_groups GROUP BY event_id
+);
-- 
cgit 1.5.1


From 86fc9b617c5eb91186a8466370ddd97f3acd34a2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 13 Apr 2015 17:03:49 +0100
Subject: For backwards compat, make state_groups.id have a type of int, not
 varchar

---
 synapse/storage/schema/full_schemas/16/state.sql | 2 +-
 synapse/storage/state.py                         | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
index 8d6b94a616..8c51610396 100644
--- a/synapse/storage/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS state_groups(
-    id VARCHAR(20) PRIMARY KEY,
+    id BIGINT UNSIGNED PRIMARY KEY,
     room_id VARCHAR(150) NOT NULL,
     event_id VARCHAR(150) NOT NULL
 );
diff --git a/synapse/storage/state.py b/synapse/storage/state.py
index 4994bacd6c..553ba9dd1f 100644
--- a/synapse/storage/state.py
+++ b/synapse/storage/state.py
@@ -93,7 +93,7 @@ class StateStore(SQLBaseStore):
 
         state_group = context.state_group
         if not state_group:
-            state_group = _make_group_id(self._clock)
+            state_group = self._state_groups_id_gen.get_next_txn(txn)
             self._simple_insert_txn(
                 txn,
                 table="state_groups",
-- 
cgit 1.5.1


From 58d83399663a080c123d2f112b4f4d84accbc638 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 14 Apr 2015 13:53:20 +0100
Subject: Add support for postgres instead of mysql. Change sql accourdingly.
 blob + varbinary -> bytea. No support for UNSIGNED or CREATE INDEX IF NOT
 EXISTS.

---
 synapse/app/homeserver.py                          |  2 +
 synapse/storage/__init__.py                        | 15 +++-----
 synapse/storage/_base.py                           |  2 +-
 synapse/storage/engines/__init__.py                |  2 +
 synapse/storage/event_federation.py                | 10 ++---
 synapse/storage/events.py                          |  4 +-
 synapse/storage/room.py                            | 34 ++++++++++++-----
 .../full_schemas/16/application_services.sql       | 10 ++---
 .../storage/schema/full_schemas/16/event_edges.sql | 26 ++++++-------
 .../schema/full_schemas/16/event_signatures.sql    | 16 ++++----
 synapse/storage/schema/full_schemas/16/im.sql      | 44 +++++++++++-----------
 synapse/storage/schema/full_schemas/16/keys.sql    |  8 ++--
 .../schema/full_schemas/16/media_repository.sql    |  8 ++--
 .../storage/schema/full_schemas/16/presence.sql    |  4 +-
 synapse/storage/schema/full_schemas/16/push.sql    | 30 +++++++--------
 .../storage/schema/full_schemas/16/redactions.sql  |  4 +-
 synapse/storage/schema/full_schemas/16/state.sql   | 10 ++---
 .../schema/full_schemas/16/transactions.sql        | 24 ++++++------
 synapse/storage/schema/full_schemas/16/users.sql   | 10 ++---
 synapse/storage/schema/schema_version.sql          | 14 +++----
 synapse/storage/stream.py                          | 16 ++++----
 21 files changed, 153 insertions(+), 140 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index a47e548d66..033011e1d7 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -373,6 +373,8 @@ def setup(config_options):
             "use_unicode": True,
             "collation": "utf8mb4_bin",
         })
+    elif name == "psycopg2":
+        pass
     elif name == "sqlite3":
         db_config.setdefault("args", {}).update({
             "cp_min": 1,
diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index b46cafd25e..272420194d 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -236,7 +236,7 @@ def _setup_new_database(cur, database_engine):
 
     cur.execute(
         database_engine.convert_param_style(
-            "REPLACE INTO schema_version (version, upgraded)"
+            "INSERT INTO schema_version (version, upgraded)"
             " VALUES (?,?)"
         ),
         (max_current_ver, False,)
@@ -432,14 +432,11 @@ def executescript(txn, schema_path):
 
 
 def _get_or_create_schema_state(txn, database_engine):
-    try:
-        # Bluntly try creating the schema_version tables.
-        schema_path = os.path.join(
-            dir_path, "schema", "schema_version.sql",
-        )
-        executescript(txn, schema_path)
-    except:
-        pass
+    # Bluntly try creating the schema_version tables.
+    schema_path = os.path.join(
+        dir_path, "schema", "schema_version.sql",
+    )
+    executescript(txn, schema_path)
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()
diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index e30514cd5e..fa5199104a 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -330,7 +330,7 @@ class SQLBaseStore(object):
                                     continue
                             raise
                 except Exception as e:
-                    logger.debug("[TXN FAIL] {%s}", name, e)
+                    logger.debug("[TXN FAIL] {%s} %s", name, e)
                     raise
                 finally:
                     end = time.time() * 1000
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 29702be923..548d4e1b42 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from .maria import MariaEngine
+from .postgres import PostgresEngine
 from .sqlite3 import Sqlite3Engine
 
 import importlib
@@ -22,6 +23,7 @@ import importlib
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
     "mysql.connector": MariaEngine,
+    "psycopg2": PostgresEngine,
 }
 
 
diff --git a/synapse/storage/event_federation.py b/synapse/storage/event_federation.py
index 79ad5ddc9c..54a3c9d805 100644
--- a/synapse/storage/event_federation.py
+++ b/synapse/storage/event_federation.py
@@ -153,7 +153,7 @@ class EventFederationStore(SQLBaseStore):
         results = self._get_prev_events_and_state(
             txn,
             event_id,
-            is_state=1,
+            is_state=True,
         )
 
         return [(e_id, h, ) for e_id, h, _ in results]
@@ -164,7 +164,7 @@ class EventFederationStore(SQLBaseStore):
         }
 
         if is_state is not None:
-            keyvalues["is_state"] = is_state
+            keyvalues["is_state"] = bool(is_state)
 
         res = self._simple_select_list_txn(
             txn,
@@ -259,7 +259,7 @@ class EventFederationStore(SQLBaseStore):
                     "event_id": event_id,
                     "prev_event_id": e_id,
                     "room_id": room_id,
-                    "is_state": 0,
+                    "is_state": False,
                 },
             )
 
@@ -397,7 +397,7 @@ class EventFederationStore(SQLBaseStore):
 
         query = (
             "SELECT prev_event_id FROM event_edges "
-            "WHERE room_id = ? AND event_id = ? AND is_state = 0 "
+            "WHERE room_id = ? AND event_id = ? AND is_state = ? "
             "LIMIT ?"
         )
 
@@ -406,7 +406,7 @@ class EventFederationStore(SQLBaseStore):
             for event_id in front:
                 txn.execute(
                     query,
-                    (room_id, event_id, limit - len(event_results))
+                    (room_id, event_id, False, limit - len(event_results))
                 )
 
                 for e_id, in txn.fetchall():
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index a2e87c27ce..9fe2effb4b 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -188,12 +188,12 @@ class EventsStore(SQLBaseStore):
                 )
 
                 sql = (
-                    "UPDATE events SET outlier = 0"
+                    "UPDATE events SET outlier = ?"
                     " WHERE event_id = ?"
                 )
                 txn.execute(
                     sql,
-                    (event.event_id,)
+                    (False, event.event_id,)
                 )
             return
 
diff --git a/synapse/storage/room.py b/synapse/storage/room.py
index a1a76280fe..48ebb33057 100644
--- a/synapse/storage/room.py
+++ b/synapse/storage/room.py
@@ -105,14 +105,12 @@ class RoomStore(SQLBaseStore):
 
             # We use non printing ascii character US (\x1F) as a separator
             sql = (
-                "SELECT r.room_id, n.name, t.topic, "
-                "group_concat(a.room_alias, '\x1F') "
-                "FROM rooms AS r "
-                "LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id "
-                "LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id "
-                "INNER JOIN room_aliases AS a ON a.room_id = r.room_id "
-                "WHERE r.is_public = ? "
-                "GROUP BY r.room_id "
+                "SELECT r.room_id, max(n.name), max(t.topic)"
+                " FROM rooms AS r"
+                " LEFT JOIN (%(topic)s) AS t ON t.room_id = r.room_id"
+                " LEFT JOIN (%(name)s) AS n ON n.room_id = r.room_id"
+                " WHERE r.is_public = ?"
+                " GROUP BY r.room_id"
             ) % {
                 "topic": topic_subquery,
                 "name": name_subquery,
@@ -120,7 +118,22 @@ class RoomStore(SQLBaseStore):
 
             txn.execute(sql, (is_public,))
 
-            return txn.fetchall()
+            rows = txn.fetchall()
+
+            for i, row in enumerate(rows):
+                room_id = row[0]
+                aliases = self._simple_select_onecol_txn(
+                    txn,
+                    table="room_aliases",
+                    keyvalues={
+                        "room_id": room_id
+                    },
+                    retcol="room_alias",
+                )
+
+                rows[i] = list(row) + [aliases]
+
+            return rows
 
         rows = yield self.runInteraction(
             "get_rooms", f
@@ -131,9 +144,10 @@ class RoomStore(SQLBaseStore):
                 "room_id": r[0],
                 "name": r[1],
                 "topic": r[2],
-                "aliases": r[3].split("\x1F"),
+                "aliases": r[3],
             }
             for r in rows
+            if r[3]  # We only return rooms that have at least one alias.
         ]
 
         defer.returnValue(ret)
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index bc709df92d..f08c5bcf76 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services(
-    id BIGINT UNSIGNED PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
     url VARCHAR(150),
     token VARCHAR(150),
     hs_token VARCHAR(150),
@@ -23,8 +23,8 @@ CREATE TABLE IF NOT EXISTS application_services(
 );
 
 CREATE TABLE IF NOT EXISTS application_services_regex(
-    id BIGINT UNSIGNED PRIMARY KEY,
-    as_id BIGINT UNSIGNED NOT NULL,
+    id BIGINT PRIMARY KEY,
+    as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
     regex VARCHAR(150),
     FOREIGN KEY(as_id) REFERENCES application_services(id)
@@ -39,10 +39,10 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids LONGBLOB NOT NULL,
+    event_ids bytea NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
-CREATE INDEX IF NOT EXISTS application_services_txns_id ON application_services_txns (
+CREATE INDEX application_services_txns_id ON application_services_txns (
     as_id
 );
diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
index bdb1109094..05d0874f0d 100644
--- a/synapse/storage/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
+CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
+CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
@@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
+CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
+CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
@@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges(
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
 
-CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
-CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
+CREATE INDEX ev_edges_id ON event_edges(event_id);
+CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
@@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth(
     UNIQUE (room_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
+CREATE INDEX room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(150) NOT NULL,
     destination VARCHAR(150) NOT NULL,
-    delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
 
-CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
+CREATE INDEX event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
@@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
+CREATE INDEX st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
 );
-CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
+CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
@@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth(
     UNIQUE (event_id, auth_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
-CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
+CREATE INDEX evauth_edges_id ON event_auth(event_id);
+CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql
index 09886f607c..4291827368 100644
--- a/synapse/storage/schema/full_schemas/16/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql
@@ -16,40 +16,40 @@
 CREATE TABLE IF NOT EXISTS event_content_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
+CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
+CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
     event_id VARCHAR(150),
     signature_name VARCHAR(150),
     key_id VARCHAR(150),
-    signature LONGBLOB,
+    signature bytea,
     UNIQUE (event_id, signature_name, key_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
+CREATE INDEX event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
     event_id VARCHAR(150),
     prev_event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
+CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 19f0f34143..a661fc160c 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -14,33 +14,33 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering BIGINT UNSIGNED PRIMARY KEY,
-    topological_ordering BIGINT UNSIGNED NOT NULL,
+    stream_ordering BIGINT PRIMARY KEY,
+    topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content LONGBLOB NOT NULL,
-    unrecognized_keys LONGBLOB,
+    content bytea NOT NULL,
+    unrecognized_keys bytea,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
-    depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
+    depth BIGINT DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
-CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
-CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
+CREATE INDEX events_stream_ordering ON events (stream_ordering);
+CREATE INDEX events_topological_ordering ON events (topological_ordering);
+CREATE INDEX events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata LONGBLOB NOT NULL,
-    json LONGBLOB NOT NULL,
+    internal_metadata bytea NOT NULL,
+    json bytea NOT NULL,
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
+CREATE INDEX event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
@@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
-CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
-CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
+CREATE INDEX state_events_room_id ON state_events (room_id);
+CREATE INDEX state_events_type ON state_events (type);
+CREATE INDEX state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
@@ -66,9 +66,9 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     UNIQUE (room_id, type, state_key)
 );
 
-CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
-CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
-CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
+CREATE INDEX current_state_events_room_id ON current_state_events (room_id);
+CREATE INDEX current_state_events_type ON current_state_events (type);
+CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
     event_id VARCHAR(150) NOT NULL,
@@ -79,8 +79,8 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
-CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
+CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
+CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
     event_id VARCHAR(150) NOT NULL,
@@ -98,7 +98,7 @@ CREATE TABLE IF NOT EXISTS topics(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
+CREATE INDEX topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
     event_id VARCHAR(150) NOT NULL,
@@ -107,7 +107,7 @@ CREATE TABLE IF NOT EXISTS room_names(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
+CREATE INDEX room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
     room_id VARCHAR(150) PRIMARY KEY NOT NULL,
@@ -121,4 +121,4 @@ CREATE TABLE IF NOT EXISTS room_hosts(
     UNIQUE (room_id, host)
 );
 
-CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
+CREATE INDEX room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index 35f141c288..459b510427 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name VARCHAR(150), -- Server name.
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
-  ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
-  tls_certificate LONGBLOB, -- DER encoded x509 certificate.
+  ts_added_ms BIGINT, -- When the certifcate was added.
+  tls_certificate bytea, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(150), -- Server name.
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
-  ts_added_ms BIGINT UNSIGNED, -- When the key was added.
-  verify_key LONGBLOB, -- NACL verification key.
+  ts_added_ms BIGINT, -- When the key was added.
+  verify_key bytea, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql
index 014bce4aeb..0e819fca38 100644
--- a/synapse/storage/schema/full_schemas/16/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/16/media_repository.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     media_id VARCHAR(150), -- The id used to refer to the media.
     media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
-    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
@@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     )
 );
 
-CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
+CREATE INDEX local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_origin VARCHAR(150), -- The remote HS the media came from.
     media_id VARCHAR(150), -- The id used to refer to the media on that server.
     media_type VARCHAR(150), -- The MIME-type of the media.
-    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(150), -- The name used to store the media on disk.
@@ -64,5 +64,5 @@ CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
      )
 );
 
-CREATE INDEX IF NOT EXISTS remote_media_cache_thumbnails_media_id
+CREATE INDEX remote_media_cache_thumbnails_media_id
     ON remote_media_cache_thumbnails (media_id);
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
index fbe5b0af6c..9c41be296e 100644
--- a/synapse/storage/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
   status_msg VARCHAR(150),
-  mtime BIGINT UNSIGNED, -- miliseconds since last state change
+  mtime BIGINT, -- miliseconds since last state change
   UNIQUE (user_id)
 );
 
@@ -37,4 +37,4 @@ CREATE TABLE IF NOT EXISTS presence_list(
   UNIQUE (user_id, observed_user_id)
 );
 
-CREATE INDEX IF NOT EXISTS presence_list_user_id ON presence_list (user_id);
+CREATE INDEX presence_list_user_id ON presence_list (user_id);
diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql
index 33300736f9..5c0c7bc201 100644
--- a/synapse/storage/schema/full_schemas/16/push.sql
+++ b/synapse/storage/schema/full_schemas/16/push.sql
@@ -22,52 +22,52 @@ CREATE TABLE IF NOT EXISTS rejections(
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
-  id BIGINT UNSIGNED PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
   app_id VARCHAR(64) NOT NULL,
   app_display_name VARCHAR(64) NOT NULL,
   device_display_name VARCHAR(128) NOT NULL,
-  pushkey VARBINARY(512) NOT NULL,
-  ts BIGINT UNSIGNED NOT NULL,
+  pushkey bytea NOT NULL,
+  ts BIGINT NOT NULL,
   lang VARCHAR(8),
-  data LONGBLOB,
+  data bytea,
   last_token TEXT,
-  last_success BIGINT UNSIGNED,
-  failing_since BIGINT UNSIGNED,
+  last_success BIGINT,
+  failing_since BIGINT,
   UNIQUE (app_id, pushkey)
 );
 
 CREATE TABLE IF NOT EXISTS push_rules (
-  id BIGINT UNSIGNED PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
-  priority_class TINYINT NOT NULL,
+  priority_class SMALLINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
   conditions VARCHAR(150) NOT NULL,
   actions VARCHAR(150) NOT NULL,
   UNIQUE(user_name, rule_id)
 );
 
-CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
+CREATE INDEX push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
   user_id VARCHAR(150),
-  filter_id BIGINT UNSIGNED,
-  filter_json LONGBLOB
+  filter_id BIGINT,
+  filter_json bytea
 );
 
-CREATE INDEX IF NOT EXISTS user_filters_by_user_id_filter_id ON user_filters(
+CREATE INDEX user_filters_by_user_id_filter_id ON user_filters(
     user_id, filter_id
 );
 
 CREATE TABLE IF NOT EXISTS push_rules_enable (
-  id BIGINT UNSIGNED PRIMARY KEY,
+  id BIGINT PRIMARY KEY,
   user_name VARCHAR(150) NOT NULL,
   rule_id VARCHAR(150) NOT NULL,
-  enabled TINYINT,
+  enabled SMALLINT,
   UNIQUE(user_name, rule_id)
 );
 
-CREATE INDEX IF NOT EXISTS push_rules_enable_user_name on push_rules_enable (user_name);
+CREATE INDEX push_rules_enable_user_name on push_rules_enable (user_name);
diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql
index b81451eab4..492fd22033 100644
--- a/synapse/storage/schema/full_schemas/16/redactions.sql
+++ b/synapse/storage/schema/full_schemas/16/redactions.sql
@@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions (
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
-CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
+CREATE INDEX redactions_event_id ON redactions (event_id);
+CREATE INDEX redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
index 8c51610396..3c54595e64 100644
--- a/synapse/storage/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS state_groups(
-    id BIGINT UNSIGNED PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
     room_id VARCHAR(150) NOT NULL,
     event_id VARCHAR(150) NOT NULL
 );
@@ -33,8 +33,8 @@ CREATE TABLE IF NOT EXISTS event_to_state_groups(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
+CREATE INDEX state_groups_id ON state_groups(id);
 
-CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group);
-CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
-CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
+CREATE INDEX state_groups_state_id ON state_groups_state(state_group);
+CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
+CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id);
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index f381e67603..bc64064936 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -16,32 +16,32 @@
 CREATE TABLE IF NOT EXISTS received_transactions(
     transaction_id VARCHAR(150),
     origin VARCHAR(150),
-    ts BIGINT UNSIGNED,
+    ts BIGINT,
     response_code INTEGER,
-    response_json LONGBLOB,
-    has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
+    response_json bytea,
+    has_been_referenced smallint default 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
 );
 
-CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
+CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
 
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
-    id BIGINT UNSIGNED PRIMARY KEY, -- This is used to apply insertion ordering
+    id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json LONGBLOB,
-    ts BIGINT UNSIGNED
+    response_json bytea,
+    ts BIGINT
 );
 
-CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
+CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
+CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
 -- So that we can do an efficient look up of all transactions that have yet to be successfully
 -- sent.
-CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code);
+CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
 
 
 -- For sent transactions only.
@@ -53,11 +53,11 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     UNIQUE (transaction_id, destination)
 );
 
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
+CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(150) PRIMARY KEY,
-    retry_last_ts BIGINT UNSIGNED,
+    retry_last_ts BIGINT,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
index d0011c04b4..267284d07d 100644
--- a/synapse/storage/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -15,17 +15,17 @@
 CREATE TABLE IF NOT EXISTS users(
     name VARCHAR(150),
     password_hash VARCHAR(150),
-    creation_ts BIGINT UNSIGNED,
-    admin BOOL DEFAULT 0 NOT NULL,
+    creation_ts BIGINT,
+    admin SMALLINT DEFAULT 0 NOT NULL,
     UNIQUE(name)
 );
 
 CREATE TABLE IF NOT EXISTS access_tokens(
-    id BIGINT UNSIGNED PRIMARY KEY,
+    id BIGINT PRIMARY KEY,
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-    last_used BIGINT UNSIGNED,
+    last_used BIGINT,
     UNIQUE(token)
 );
 
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT UNSIGNED NOT NULL
+    last_seen BIGINT NOT NULL
 );
 
 CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index e7fa6fe569..d9494611e0 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -14,16 +14,14 @@
  */
 
 CREATE TABLE IF NOT EXISTS schema_version(
-    `Lock` CHAR(1) NOT NULL DEFAULT 'X' UNIQUE,  -- Makes sure this table only has one row.
-    `version` INTEGER NOT NULL,
-    `upgraded` BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
-    CHECK (`Lock`='X')
+    Lock CHAR(1) NOT NULL DEFAULT 'X' UNIQUE,  -- Makes sure this table only has one row.
+    version INTEGER NOT NULL,
+    upgraded BOOL NOT NULL,  -- Whether we reached this version from an upgrade or an initial schema.
+    CHECK (Lock='X')
 );
 
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
-    `version` INTEGER NOT NULL,
-    `file` VARCHAR(150) NOT NULL,
+    version INTEGER NOT NULL,
+    file VARCHAR(150) NOT NULL,
     UNIQUE(version, file)
 );
-
-CREATE INDEX IF NOT EXISTS schema_deltas_ver ON applied_schema_deltas(version);
diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 57c2e4dfeb..df6de7cbcd 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -240,7 +240,7 @@ class StreamStore(SQLBaseStore):
 
         sql = (
             "SELECT e.event_id, e.stream_ordering FROM events AS e WHERE "
-            "(e.outlier = 0 AND (room_id IN (%(current)s)) OR "
+            "(e.outlier = ? AND (room_id IN (%(current)s)) OR "
             "(event_id IN (%(invites)s))) "
             "AND e.stream_ordering > ? AND e.stream_ordering <= ? "
             "ORDER BY stream_ordering ASC LIMIT %(limit)d "
@@ -251,7 +251,7 @@ class StreamStore(SQLBaseStore):
         }
 
         def f(txn):
-            txn.execute(sql, (user_id, user_id, from_id.stream, to_id.stream,))
+            txn.execute(sql, (False, user_id, user_id, from_id.stream, to_id.stream,))
 
             rows = self.cursor_to_dict(txn)
 
@@ -283,7 +283,7 @@ class StreamStore(SQLBaseStore):
         # Tokens really represent positions between elements, but we use
         # the convention of pointing to the event before the gap. Hence
         # we have a bit of asymmetry when it comes to equalities.
-        args = [room_id]
+        args = [False, room_id]
         if direction == 'b':
             order = "DESC"
             bounds = _StreamToken.parse(from_key).upper_bound()
@@ -307,7 +307,7 @@ class StreamStore(SQLBaseStore):
 
         sql = (
             "SELECT * FROM events"
-            " WHERE outlier = 0 AND room_id = ? AND %(bounds)s"
+            " WHERE outlier = ? AND room_id = ? AND %(bounds)s"
             " ORDER BY topological_ordering %(order)s,"
             " stream_ordering %(order)s %(limit)s"
         ) % {
@@ -358,7 +358,7 @@ class StreamStore(SQLBaseStore):
             sql = (
                 "SELECT stream_ordering, topological_ordering, event_id"
                 " FROM events"
-                " WHERE room_id = ? AND stream_ordering <= ? AND outlier = 0"
+                " WHERE room_id = ? AND stream_ordering <= ? AND outlier = ?"
                 " ORDER BY topological_ordering DESC, stream_ordering DESC"
                 " LIMIT ?"
             )
@@ -368,17 +368,17 @@ class StreamStore(SQLBaseStore):
                 "SELECT stream_ordering, topological_ordering, event_id"
                 " FROM events"
                 " WHERE room_id = ? AND stream_ordering > ?"
-                " AND stream_ordering <= ? AND outlier = 0"
+                " AND stream_ordering <= ? AND outlier = ?"
                 " ORDER BY topological_ordering DESC, stream_ordering DESC"
                 " LIMIT ?"
             )
 
         def get_recent_events_for_room_txn(txn):
             if from_token is None:
-                txn.execute(sql, (room_id, end_token.stream, limit,))
+                txn.execute(sql, (room_id, end_token.stream, False, limit,))
             else:
                 txn.execute(sql, (
-                    room_id, from_token.stream, end_token.stream, limit
+                    room_id, from_token.stream, end_token.stream, False, limit
                 ))
 
             rows = self.cursor_to_dict(txn)
-- 
cgit 1.5.1


From 5a95cd444297dfc2aab98259d6fc6a07bed837b9 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 14 Apr 2015 13:54:09 +0100
Subject: Rename user_ips.user -> user_id

---
 synapse/storage/__init__.py                      | 4 ++--
 synapse/storage/schema/delta/16/users.sql        | 6 +++---
 synapse/storage/schema/full_schemas/16/users.sql | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/__init__.py b/synapse/storage/__init__.py
index 272420194d..995114e405 100644
--- a/synapse/storage/__init__.py
+++ b/synapse/storage/__init__.py
@@ -107,7 +107,7 @@ class DataStore(RoomMemberStore, RoomStore,
         yield self._simple_upsert(
             "user_ips",
             keyvalues={
-                "user": user.to_string(),
+                "user_id": user.to_string(),
                 "access_token": access_token,
                 "ip": ip,
                 "user_agent": user_agent,
@@ -122,7 +122,7 @@ class DataStore(RoomMemberStore, RoomStore,
     def get_user_ip_and_agents(self, user):
         return self._simple_select_list(
             table="user_ips",
-            keyvalues={"user": user.to_string()},
+            keyvalues={"user_id": user.to_string()},
             retcols=[
                 "device_id", "access_token", "ip", "user_agent", "last_seen"
             ],
diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql
index f04705cd77..db27bdca02 100644
--- a/synapse/storage/schema/delta/16/users.sql
+++ b/synapse/storage/schema/delta/16/users.sql
@@ -36,7 +36,7 @@ ALTER TABLE new_users RENAME TO users;
 
 -- Remove UNIQUE constraint from `user_ips` table
 CREATE TABLE IF NOT EXISTS new_user_ips (
-    user VARCHAR(150) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
     access_token VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
@@ -51,6 +51,6 @@ DROP TABLE user_ips;
 
 ALTER TABLE new_user_ips RENAME TO user_ips;
 
-CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
+CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user_id);
+CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user_id, access_token, ip);
 
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
index 267284d07d..006b249fc0 100644
--- a/synapse/storage/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS access_tokens(
 );
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user VARCHAR(150) NOT NULL,
+    user_id VARCHAR(150) NOT NULL,
     access_token VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
@@ -38,5 +38,5 @@ CREATE TABLE IF NOT EXISTS user_ips (
     last_seen BIGINT NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
-CREATE INDEX IF NOT EXISTS user_ips_user_ip ON user_ips(user, access_token, ip);
+CREATE INDEX user_ips_user ON user_ips(user_id);
+CREATE INDEX user_ips_user_ip ON user_ips(user_id, access_token, ip);
-- 
cgit 1.5.1


From cce5d057d34ac6fc953d1987bbce3bce2ce42bd8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 11:43:25 +0100
Subject: Add index on events (topological_ordering, stream_ordering) to help
 with ORDER BY clauses

---
 synapse/storage/schema/delta/16/events_order_index.sql | 1 +
 synapse/storage/schema/full_schemas/16/im.sql          | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 synapse/storage/schema/delta/16/events_order_index.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql
new file mode 100644
index 0000000000..c015414663
--- /dev/null
+++ b/synapse/storage/schema/delta/16/events_order_index.sql
@@ -0,0 +1 @@
+CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
\ No newline at end of file
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index a661fc160c..b8faed5066 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -29,6 +29,7 @@ CREATE TABLE IF NOT EXISTS events(
 
 CREATE INDEX events_stream_ordering ON events (stream_ordering);
 CREATE INDEX events_topological_ordering ON events (topological_ordering);
+CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
 CREATE INDEX events_room_id ON events (room_id);
 
 
-- 
cgit 1.5.1


From cf04cedf216a21f36a476703470b7ffe1a727967 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 16:53:47 +0100
Subject: Change full_schemas/11 to work with postgres

---
 .../storage/schema/delta/16/events_order_index.sql |  5 ++-
 .../storage/schema/full_schemas/11/event_edges.sql | 26 +++++------
 .../schema/full_schemas/11/event_signatures.sql    | 16 +++----
 synapse/storage/schema/full_schemas/11/im.sql      | 50 +++++++++++-----------
 synapse/storage/schema/full_schemas/11/keys.sql    |  8 ++--
 .../schema/full_schemas/11/media_repository.sql    |  6 +--
 .../storage/schema/full_schemas/11/presence.sql    |  2 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  4 +-
 synapse/storage/schema/full_schemas/11/state.sql   | 14 +++---
 .../schema/full_schemas/11/transactions.sql        | 24 +++++------
 synapse/storage/schema/full_schemas/11/users.sql   | 10 ++---
 synapse/storage/schema/full_schemas/16/im.sql      |  5 ++-
 synapse/storage/schema/full_schemas/16/state.sql   |  4 +-
 13 files changed, 90 insertions(+), 84 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/events_order_index.sql b/synapse/storage/schema/delta/16/events_order_index.sql
index c015414663..a48f215170 100644
--- a/synapse/storage/schema/delta/16/events_order_index.sql
+++ b/synapse/storage/schema/delta/16/events_order_index.sql
@@ -1 +1,4 @@
-CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
\ No newline at end of file
+CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
+CREATE INDEX events_order_room ON events (
+    room_id, topological_ordering, stream_ordering
+);
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index bdb1109094..05d0874f0d 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS event_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_extrem_room ON event_forward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_extrem_id ON event_forward_extremities(event_id);
+CREATE INDEX ev_extrem_room ON event_forward_extremities(room_id);
+CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
@@ -29,8 +29,8 @@ CREATE TABLE IF NOT EXISTS event_backward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS ev_b_extrem_room ON event_backward_extremities(room_id);
-CREATE INDEX IF NOT EXISTS ev_b_extrem_id ON event_backward_extremities(event_id);
+CREATE INDEX ev_b_extrem_room ON event_backward_extremities(room_id);
+CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
@@ -41,8 +41,8 @@ CREATE TABLE IF NOT EXISTS event_edges(
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
 
-CREATE INDEX IF NOT EXISTS ev_edges_id ON event_edges(event_id);
-CREATE INDEX IF NOT EXISTS ev_edges_prev_id ON event_edges(prev_event_id);
+CREATE INDEX ev_edges_id ON event_edges(event_id);
+CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
@@ -51,17 +51,17 @@ CREATE TABLE IF NOT EXISTS room_depth(
     UNIQUE (room_id)
 );
 
-CREATE INDEX IF NOT EXISTS room_depth_room ON room_depth(room_id);
+CREATE INDEX room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
     event_id VARCHAR(150) NOT NULL,
     destination VARCHAR(150) NOT NULL,
-    delivered_ts BIGINT UNSIGNED DEFAULT 0, -- or 0 if not delivered
+    delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
 
-CREATE INDEX IF NOT EXISTS event_destinations_id ON event_destinations(event_id);
+CREATE INDEX event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
@@ -72,10 +72,10 @@ CREATE TABLE IF NOT EXISTS state_forward_extremities(
     UNIQUE (event_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS st_extrem_keys ON state_forward_extremities(
+CREATE INDEX st_extrem_keys ON state_forward_extremities(
     room_id, type, state_key
 );
-CREATE INDEX IF NOT EXISTS st_extrem_id ON state_forward_extremities(event_id);
+CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
@@ -85,5 +85,5 @@ CREATE TABLE IF NOT EXISTS event_auth(
     UNIQUE (event_id, auth_id, room_id)
 );
 
-CREATE INDEX IF NOT EXISTS evauth_edges_id ON event_auth(event_id);
-CREATE INDEX IF NOT EXISTS evauth_edges_auth_id ON event_auth(auth_id);
+CREATE INDEX evauth_edges_id ON event_auth(event_id);
+CREATE INDEX evauth_edges_auth_id ON event_auth(auth_id);
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 09886f607c..4291827368 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -16,40 +16,40 @@
 CREATE TABLE IF NOT EXISTS event_content_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_content_hashes_id ON event_content_hashes(event_id);
+CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
     event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_reference_hashes_id ON event_reference_hashes(event_id);
+CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
     event_id VARCHAR(150),
     signature_name VARCHAR(150),
     key_id VARCHAR(150),
-    signature LONGBLOB,
+    signature bytea,
     UNIQUE (event_id, signature_name, key_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_signatures_id ON event_signatures(event_id);
+CREATE INDEX event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
     event_id VARCHAR(150),
     prev_event_id VARCHAR(150),
     algorithm VARCHAR(150),
-    hash LONGBLOB,
+    hash bytea,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
 
-CREATE INDEX IF NOT EXISTS event_edge_hashes_id ON event_edge_hashes(event_id);
+CREATE INDEX event_edge_hashes_id ON event_edge_hashes(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index 9c47f51742..e9e09214d7 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -15,32 +15,32 @@
 
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
-    topological_ordering BIGINT UNSIGNED NOT NULL,
+    topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content LONGBLOB NOT NULL,
-    unrecognized_keys LONGBLOB,
+    content bytea NOT NULL,
+    unrecognized_keys bytea,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
-    depth BIGINT UNSIGNED DEFAULT 0 NOT NULL,
+    depth BIGINT DEFAULT 0 NOT NULL,
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS events_stream_ordering ON events (stream_ordering);
-CREATE INDEX IF NOT EXISTS events_topological_ordering ON events (topological_ordering);
-CREATE INDEX IF NOT EXISTS events_room_id ON events (room_id);
+CREATE INDEX events_stream_ordering ON events (stream_ordering);
+CREATE INDEX events_topological_ordering ON events (topological_ordering);
+CREATE INDEX events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata LONGBLOB NOT NULL,
-    json LONGBLOB NOT NULL,
+    internal_metadata bytea NOT NULL,
+    json bytea NOT NULL,
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS event_json_room_id ON event_json(room_id);
+CREATE INDEX event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
@@ -52,9 +52,9 @@ CREATE TABLE IF NOT EXISTS state_events(
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS state_events_room_id ON state_events (room_id);
-CREATE INDEX IF NOT EXISTS state_events_type ON state_events (type);
-CREATE INDEX IF NOT EXISTS state_events_state_key ON state_events (state_key);
+CREATE INDEX state_events_room_id ON state_events (room_id);
+CREATE INDEX state_events_type ON state_events (type);
+CREATE INDEX state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
@@ -65,10 +65,10 @@ CREATE TABLE IF NOT EXISTS current_state_events(
     UNIQUE (room_id, type, state_key)
 );
 
-CREATE INDEX IF NOT EXISTS curr_events_event_id ON current_state_events (event_id);
-CREATE INDEX IF NOT EXISTS current_state_events_room_id ON current_state_events (room_id);
-CREATE INDEX IF NOT EXISTS current_state_events_type ON current_state_events (type);
-CREATE INDEX IF NOT EXISTS current_state_events_state_key ON current_state_events (state_key);
+CREATE INDEX curr_events_event_id ON current_state_events (event_id);
+CREATE INDEX current_state_events_room_id ON current_state_events (room_id);
+CREATE INDEX current_state_events_type ON current_state_events (type);
+CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
     event_id VARCHAR(150) NOT NULL,
@@ -78,9 +78,9 @@ CREATE TABLE IF NOT EXISTS room_memberships(
     membership VARCHAR(150) NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS room_memberships_event_id ON room_memberships (event_id);
-CREATE INDEX IF NOT EXISTS room_memberships_room_id ON room_memberships (room_id);
-CREATE INDEX IF NOT EXISTS room_memberships_user_id ON room_memberships (user_id);
+CREATE INDEX room_memberships_event_id ON room_memberships (event_id);
+CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
+CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
     event_id VARCHAR(150) NOT NULL,
@@ -96,8 +96,8 @@ CREATE TABLE IF NOT EXISTS topics(
     topic TEXT NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS topics_event_id ON topics(event_id);
-CREATE INDEX IF NOT EXISTS topics_room_id ON topics(room_id);
+CREATE INDEX topics_event_id ON topics(event_id);
+CREATE INDEX topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
     event_id VARCHAR(150) NOT NULL,
@@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS room_names(
     name TEXT NOT NULL
 );
 
-CREATE INDEX IF NOT EXISTS room_names_event_id ON room_names(event_id);
-CREATE INDEX IF NOT EXISTS room_names_room_id ON room_names(room_id);
+CREATE INDEX room_names_event_id ON room_names(event_id);
+CREATE INDEX room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
     room_id VARCHAR(150) PRIMARY KEY NOT NULL,
@@ -120,4 +120,4 @@ CREATE TABLE IF NOT EXISTS room_hosts(
     UNIQUE (room_id, host)
 );
 
-CREATE INDEX IF NOT EXISTS room_hosts_room_id ON room_hosts (room_id);
+CREATE INDEX room_hosts_room_id ON room_hosts (room_id);
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 35f141c288..459b510427 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS server_tls_certificates(
   server_name VARCHAR(150), -- Server name.
   fingerprint VARCHAR(150), -- Certificate fingerprint.
   from_server VARCHAR(150), -- Which key server the certificate was fetched from.
-  ts_added_ms BIGINT UNSIGNED, -- When the certifcate was added.
-  tls_certificate LONGBLOB, -- DER encoded x509 certificate.
+  ts_added_ms BIGINT, -- When the certifcate was added.
+  tls_certificate bytea, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
@@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name VARCHAR(150), -- Server name.
   key_id VARCHAR(150), -- Key version.
   from_server VARCHAR(150), -- Which key server the key was fetched form.
-  ts_added_ms BIGINT UNSIGNED, -- When the key was added.
-  verify_key LONGBLOB, -- NACL verification key.
+  ts_added_ms BIGINT, -- When the key was added.
+  verify_key bytea, -- NACL verification key.
   UNIQUE (server_name, key_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 134e7fbcec..6e0ee0db41 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -17,7 +17,7 @@ CREATE TABLE IF NOT EXISTS local_media_repository (
     media_id VARCHAR(150), -- The id used to refer to the media.
     media_type VARCHAR(150), -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
-    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     user_id VARCHAR(150), -- The user who uploaded the file.
     UNIQUE (media_id)
@@ -35,14 +35,14 @@ CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
     )
 );
 
-CREATE INDEX IF NOT EXISTS local_media_repository_thumbnails_media_id
+CREATE INDEX local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
     media_origin VARCHAR(150), -- The remote HS the media came from.
     media_id VARCHAR(150), -- The id used to refer to the media on that server.
     media_type VARCHAR(150), -- The MIME-type of the media.
-    created_ts BIGINT UNSIGNED, -- When the content was uploaded in ms.
+    created_ts BIGINT, -- When the content was uploaded in ms.
     upload_name VARCHAR(150), -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
     filesystem_id VARCHAR(150), -- The name used to store the media on disk.
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index c617ebea73..fce324b890 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS presence(
   user_id VARCHAR(150) NOT NULL,
   state VARCHAR(20),
   status_msg VARCHAR(150),
-  mtime BIGINT UNSIGNED -- miliseconds since last state change
+  mtime BIGINT -- miliseconds since last state change
 );
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index b81451eab4..492fd22033 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -18,5 +18,5 @@ CREATE TABLE IF NOT EXISTS redactions (
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS redactions_event_id ON redactions (event_id);
-CREATE INDEX IF NOT EXISTS redactions_redacts ON redactions (redacts);
+CREATE INDEX redactions_event_id ON redactions (event_id);
+CREATE INDEX redactions_redacts ON redactions (redacts);
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 737c3e35c7..62c20819fd 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -14,13 +14,13 @@
  */
 
 CREATE TABLE IF NOT EXISTS state_groups(
-    id VARCHAR(20) PRIMARY KEY,
+    id INTEGER PRIMARY KEY,
     room_id VARCHAR(150) NOT NULL,
     event_id VARCHAR(150) NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
-    state_group VARCHAR(20) NOT NULL,
+    state_group INTEGER NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     state_key VARCHAR(150) NOT NULL,
@@ -29,12 +29,12 @@ CREATE TABLE IF NOT EXISTS state_groups_state(
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
     event_id VARCHAR(150) NOT NULL,
-    state_group VARCHAR(150) NOT NULL,
+    state_group INTEGER NOT NULL,
     UNIQUE (event_id)
 );
 
-CREATE INDEX IF NOT EXISTS state_groups_id ON state_groups(id);
+CREATE INDEX state_groups_id ON state_groups(id);
 
-CREATE INDEX IF NOT EXISTS state_groups_state_id ON state_groups_state(state_group);
-CREATE INDEX IF NOT EXISTS state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
-CREATE INDEX IF NOT EXISTS event_to_state_groups_id ON event_to_state_groups(event_id);
\ No newline at end of file
+CREATE INDEX state_groups_state_id ON state_groups_state(state_group);
+CREATE INDEX state_groups_state_tuple ON state_groups_state(room_id, type, state_key);
+CREATE INDEX event_to_state_groups_id ON event_to_state_groups(event_id);
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index c2fab10aa0..f01d9eba32 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -16,14 +16,14 @@
 CREATE TABLE IF NOT EXISTS received_transactions(
     transaction_id VARCHAR(150),
     origin VARCHAR(150),
-    ts BIGINT UNSIGNED,
+    ts BIGINT,
     response_code INTEGER,
-    response_json LONGBLOB,
-    has_been_referenced BOOL default 0, -- Whether thishas been referenced by a prev_tx
+    response_json bytea,
+    has_been_referenced SMALLINT DEFAULT 0, -- Whether thishas been referenced by a prev_tx
     UNIQUE (transaction_id, origin)
 );
 
-CREATE INDEX IF NOT EXISTS transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
+CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_referenced);-- WHERE has_been_referenced = 0;
 
 
 -- Stores what transactions we've sent, what their response was (if we got one) and whether we have
@@ -33,15 +33,15 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json LONGBLOB,
-    ts BIGINT UNSIGNED
+    response_json bytea,
+    ts BIGINT
 );
 
-CREATE INDEX IF NOT EXISTS sent_transaction_dest ON sent_transactions(destination);
-CREATE INDEX IF NOT EXISTS sent_transaction_txn_id ON sent_transactions(transaction_id);
+CREATE INDEX sent_transaction_dest ON sent_transactions(destination);
+CREATE INDEX sent_transaction_txn_id ON sent_transactions(transaction_id);
 -- So that we can do an efficient look up of all transactions that have yet to be successfully
 -- sent.
-CREATE INDEX IF NOT EXISTS sent_transaction_sent ON sent_transactions(response_code);
+CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
 
 
 -- For sent transactions only.
@@ -52,12 +52,12 @@ CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     pdu_origin VARCHAR(150)
 );
 
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
-CREATE INDEX IF NOT EXISTS transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
+CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
+CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
     destination VARCHAR(150) PRIMARY KEY,
-    retry_last_ts BIGINT UNSIGNED,
+    retry_last_ts BIGINT,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 0ddfccd410..2edc13c3b7 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -16,8 +16,8 @@ CREATE TABLE IF NOT EXISTS users(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     name VARCHAR(150),
     password_hash VARCHAR(150),
-    creation_ts BIGINT UNSIGNED,
-    admin BOOL DEFAULT 0 NOT NULL,
+    creation_ts BIGINT,
+    admin SMALLINT DEFAULT 0 NOT NULL,
     UNIQUE(name)
 );
 
@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS access_tokens(
     user_id VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     token VARCHAR(150) NOT NULL,
-    last_used BIGINT UNSIGNED,
+    last_used BIGINT,
     UNIQUE(token)
 );
 
@@ -36,8 +36,8 @@ CREATE TABLE IF NOT EXISTS user_ips (
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
     user_agent VARCHAR(150) NOT NULL,
-    last_seen BIGINT UNSIGNED NOT NULL,
+    last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
 );
 
-CREATE INDEX IF NOT EXISTS user_ips_user ON user_ips(user);
+CREATE INDEX user_ips_user ON user_ips(user);
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index b8faed5066..17e4c949b9 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -14,7 +14,7 @@
  */
 
 CREATE TABLE IF NOT EXISTS events(
-    stream_ordering BIGINT PRIMARY KEY,
+    stream_ordering INTEGER PRIMARY KEY,
     topological_ordering BIGINT NOT NULL,
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
@@ -31,6 +31,9 @@ CREATE INDEX events_stream_ordering ON events (stream_ordering);
 CREATE INDEX events_topological_ordering ON events (topological_ordering);
 CREATE INDEX events_order ON events (topological_ordering, stream_ordering);
 CREATE INDEX events_room_id ON events (room_id);
+CREATE INDEX events_order_room ON events (
+    room_id, topological_ordering, stream_ordering
+);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
index 3c54595e64..b0cd5ee75a 100644
--- a/synapse/storage/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS state_groups(
 );
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
-    state_group VARCHAR(20) NOT NULL,
+    state_group BIGINT NOT NULL,
     room_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     state_key VARCHAR(150) NOT NULL,
@@ -29,7 +29,7 @@ CREATE TABLE IF NOT EXISTS state_groups_state(
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
     event_id VARCHAR(150) NOT NULL,
-    state_group VARCHAR(150) NOT NULL,
+    state_group BIGINT NOT NULL,
     UNIQUE (event_id)
 );
 
-- 
cgit 1.5.1


From ffad75bd6284873c27efb2cfdfdcf9f909eb9db3 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Wed, 15 Apr 2015 17:00:50 +0100
Subject: Remove mysql/maria support

---
 scripts/port_from_sqlite_to_postgres.py            | 373 ++++++++++++++++++++
 scripts/port_to_maria.py                           | 383 ---------------------
 synapse/app/homeserver.py                          |   9 +-
 synapse/storage/engines/__init__.py                |   2 -
 synapse/storage/engines/maria.py                   |  50 ---
 .../storage/schema/delta/16/unique_constraints.sql |   2 +-
 6 files changed, 375 insertions(+), 444 deletions(-)
 create mode 100644 scripts/port_from_sqlite_to_postgres.py
 delete mode 100644 scripts/port_to_maria.py
 delete mode 100644 synapse/storage/engines/maria.py

(limited to 'synapse/storage/schema')

diff --git a/scripts/port_from_sqlite_to_postgres.py b/scripts/port_from_sqlite_to_postgres.py
new file mode 100644
index 0000000000..4b3fd9e529
--- /dev/null
+++ b/scripts/port_from_sqlite_to_postgres.py
@@ -0,0 +1,373 @@
+# -*- coding: utf-8 -*-
+# Copyright 2015 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from twisted.internet import defer, reactor
+from twisted.enterprise import adbapi
+
+from synapse.storage._base import LoggingTransaction, SQLBaseStore
+from synapse.storage.engines import create_engine
+
+import argparse
+import itertools
+import logging
+import types
+import yaml
+
+
+logger = logging.getLogger("port_from_sqlite_to_postgres")
+
+
+BINARY_COLUMNS = {
+    "event_content_hashes": ["hash"],
+    "event_reference_hashes": ["hash"],
+    "event_signatures": ["signature"],
+    "event_edge_hashes": ["hash"],
+    "events": ["content", "unrecognized_keys"],
+    "event_json": ["internal_metadata", "json"],
+    "application_services_txns": ["event_ids"],
+    "received_transactions": ["response_json"],
+    "sent_transactions": ["response_json"],
+    "server_tls_certificates": ["tls_certificate"],
+    "server_signature_keys": ["verify_key"],
+    "pushers": ["pushkey", "data"],
+    "user_filters": ["filter_json"],
+}
+
+UNICODE_COLUMNS = {
+    "events": ["content", "unrecognized_keys"],
+    "event_json": ["internal_metadata", "json"],
+    "users": ["password_hash"],
+}
+
+
+BOOLEAN_COLUMNS = {
+    "events": ["processed", "outlier"],
+    "rooms": ["is_public"],
+    "event_edges": ["is_state"],
+    "presence_list": ["accepted"],
+}
+
+
+APPEND_ONLY_TABLES = [
+    "event_content_hashes",
+    "event_reference_hashes",
+    "event_signatures",
+    "event_edge_hashes",
+    "events",
+    "event_json",
+    "state_events",
+    "room_memberships",
+    "feedback",
+    "topics",
+    "room_names",
+    "rooms",
+    "local_media_repository",
+    "local_media_repository_thumbnails",
+    "remote_media_cache",
+    "remote_media_cache_thumbnails",
+    "redactions",
+    "event_edges",
+    "event_auth",
+    "received_transactions",
+    "sent_transactions",
+    "transaction_id_to_pdu",
+    "users",
+    "state_groups",
+    "state_groups_state",
+    "event_to_state_groups",
+    "rejections",
+]
+
+
+class Store(object):
+    def __init__(self, db_pool, engine):
+        self.db_pool = db_pool
+        self.database_engine = engine
+
+    _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
+    _simple_insert = SQLBaseStore.__dict__["_simple_insert"]
+
+    _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
+    _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
+    _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
+    _simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"]
+
+    _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
+    _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
+
+    _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"]
+
+    def runInteraction(self, desc, func, *args, **kwargs):
+        def r(conn):
+            try:
+                i = 0
+                N = 5
+                while True:
+                    try:
+                        txn = conn.cursor()
+                        return func(
+                            LoggingTransaction(txn, desc, self.database_engine),
+                            *args, **kwargs
+                        )
+                    except self.database_engine.module.DatabaseError as e:
+                        if self.database_engine.is_deadlock(e):
+                            logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N)
+                            if i < N:
+                                i += 1
+                                conn.rollback()
+                                continue
+                        raise
+            except Exception as e:
+                logger.debug("[TXN FAIL] {%s}", desc, e)
+                raise
+
+        return self.db_pool.runWithConnection(r)
+
+    def insert_many_txn(self, txn, table, headers, rows):
+        sql = "INSERT INTO %s (%s) VALUES (%s)" % (
+            table,
+            ", ".join(k for k in headers),
+            ", ".join("%s" for _ in headers)
+        )
+
+        try:
+            txn.executemany(sql, rows)
+        except:
+            logger.exception(
+                "Failed to insert: %s",
+                table,
+            )
+            raise
+
+
+
+def chunks(n):
+    for i in itertools.count(0, n):
+        yield range(i, i+n)
+
+
+@defer.inlineCallbacks
+def handle_table(table, sqlite_store, postgres_store):
+    if table in APPEND_ONLY_TABLES:
+        # It's safe to just carry on inserting.
+        next_chunk = yield postgres_store._simple_select_one_onecol(
+            table="port_from_sqlite3",
+            keyvalues={"table_name": table},
+            retcol="rowid",
+            allow_none=True,
+        )
+
+        if next_chunk is None:
+            yield postgres_store._simple_insert(
+                table="port_from_sqlite3",
+                values={"table_name": table, "rowid": 0}
+            )
+
+            next_chunk = 0
+    else:
+        def delete_all(txn):
+            txn.execute(
+                "DELETE FROM port_from_sqlite3 WHERE table_name = %s",
+                (table,)
+            )
+            txn.execute("TRUNCATE %s CASCADE" % (table,))
+            postgres_store._simple_insert_txn(
+                txn,
+                table="port_from_sqlite3",
+                values={"table_name": table, "rowid": 0}
+            )
+
+        yield postgres_store.runInteraction(
+            "delete_non_append_only", delete_all
+        )
+
+        next_chunk = 0
+
+    logger.info("next_chunk for %s: %d", table, next_chunk)
+
+    N = 5000
+
+    select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
+
+    uni_col_names = UNICODE_COLUMNS.get(table, [])
+    bool_col_names = BOOLEAN_COLUMNS.get(table, [])
+    bin_col_names = BINARY_COLUMNS.get(table, [])
+
+    while True:
+        def r(txn):
+            txn.execute(select, (next_chunk, N,))
+            rows = txn.fetchall()
+            headers = [column[0] for column in txn.description]
+
+            return headers, rows
+
+        headers, rows = yield sqlite_store.runInteraction("select", r)
+
+        logger.info("Got %d rows for %s", len(rows), table)
+
+        if rows:
+            uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names]
+            bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]
+            bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names]
+            next_chunk = rows[-1][0] + 1
+
+            def conv(j, col):
+                if j in uni_cols:
+                    col = sqlite_store.database_engine.load_unicode(col)
+                if j in bool_cols:
+                    return bool(col)
+
+                if j in bin_cols:
+                    if isinstance(col, types.UnicodeType):
+                        col = buffer(col.encode("utf8"))
+
+                return col
+
+            for i, row in enumerate(rows):
+                rows[i] = tuple(
+                    postgres_store.database_engine.encode_parameter(
+                        conv(j, col)
+                    )
+                    for j, col in enumerate(row)
+                    if j > 0
+                )
+
+            def ins(txn):
+                postgres_store.insert_many_txn(txn, table, headers[1:], rows)
+
+                postgres_store._simple_update_one_txn(
+                    txn,
+                    table="port_from_sqlite3",
+                    keyvalues={"table_name": table},
+                    updatevalues={"rowid": next_chunk},
+                )
+
+            yield postgres_store.runInteraction("insert_many", ins)
+        else:
+            return
+
+
+def setup_db(db_config, database_engine):
+    db_conn = database_engine.module.connect(
+        **{
+            k: v for k, v in db_config.get("args", {}).items()
+            if not k.startswith("cp_")
+        }
+    )
+
+    database_engine.prepare_database(db_conn)
+
+    db_conn.commit()
+
+
+@defer.inlineCallbacks
+def main(sqlite_config, postgress_config):
+    try:
+        sqlite_db_pool = adbapi.ConnectionPool(
+            sqlite_config["name"],
+            **sqlite_config["args"]
+        )
+
+        postgres_db_pool = adbapi.ConnectionPool(
+            postgress_config["name"],
+            **postgress_config["args"]
+        )
+
+        sqlite_engine = create_engine("sqlite3")
+        postgres_engine = create_engine("psycopg2")
+
+        sqlite_store = Store(sqlite_db_pool, sqlite_engine)
+        postgres_store = Store(postgres_db_pool, postgres_engine)
+
+        # Step 1. Set up databases.
+        logger.info("Preparing sqlite database...")
+        setup_db(sqlite_config, sqlite_engine)
+
+        logger.info("Preparing postgres database...")
+        setup_db(postgress_config, postgres_engine)
+
+        # Step 2. Get tables.
+        logger.info("Fetching tables...")
+        tables = yield sqlite_store._simple_select_onecol(
+            table="sqlite_master",
+            keyvalues={
+                "type": "table",
+            },
+            retcol="name",
+        )
+
+        logger.info("Found %d tables", len(tables))
+
+        def create_port_table(txn):
+            txn.execute(
+                "CREATE TABLE port_from_sqlite3 ("
+                " table_name varchar(100) NOT NULL UNIQUE,"
+                " rowid bigint NOT NULL"
+                ")"
+            )
+
+        try:
+            yield postgres_store.runInteraction(
+                "create_port_table", create_port_table
+            )
+        except Exception as e:
+            logger.info("Failed to create port table: %s", e)
+
+        # Process tables.
+        yield defer.gatherResults(
+            [
+                handle_table(table, sqlite_store, postgres_store)
+                for table in tables
+                if table not in ["schema_version", "applied_schema_deltas"]
+                and not table.startswith("sqlite_")
+            ],
+            consumeErrors=True,
+        )
+
+    except:
+        logger.exception("")
+    finally:
+        reactor.stop()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--sqlite-database")
+    parser.add_argument(
+        "--postgres-config", type=argparse.FileType('r'),
+    )
+
+    args = parser.parse_args()
+    logging.basicConfig(level=logging.INFO)
+
+    sqlite_config = {
+        "name": "sqlite3",
+        "args": {
+            "database": args.sqlite_database,
+            "cp_min": 1,
+            "cp_max": 1,
+            "check_same_thread": False,
+        },
+    }
+
+    postgres_config = yaml.safe_load(args.postgres_config)
+
+    reactor.callWhenRunning(
+        main,
+        sqlite_config=sqlite_config,
+        postgres_config=postgres_config,
+    )
+
+    reactor.run()
diff --git a/scripts/port_to_maria.py b/scripts/port_to_maria.py
deleted file mode 100644
index 0d7ba92357..0000000000
--- a/scripts/port_to_maria.py
+++ /dev/null
@@ -1,383 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from twisted.internet import defer, reactor
-from twisted.enterprise import adbapi
-
-from synapse.storage._base import LoggingTransaction, SQLBaseStore
-from synapse.storage.engines import create_engine
-
-import argparse
-import itertools
-import logging
-import types
-import yaml
-
-
-logger = logging.getLogger("port_to_maria")
-
-
-BINARY_COLUMNS = {
-    "event_content_hashes": ["hash"],
-    "event_reference_hashes": ["hash"],
-    "event_signatures": ["signature"],
-    "event_edge_hashes": ["hash"],
-    "events": ["content", "unrecognized_keys"],
-    "event_json": ["internal_metadata", "json"],
-    "application_services_txns": ["event_ids"],
-    "received_transactions": ["response_json"],
-    "sent_transactions": ["response_json"],
-    "server_tls_certificates": ["tls_certificate"],
-    "server_signature_keys": ["verify_key"],
-    "pushers": ["pushkey", "data"],
-    "user_filters": ["filter_json"],
-}
-
-UNICODE_COLUMNS = {
-    "events": ["content", "unrecognized_keys"],
-    "event_json": ["internal_metadata", "json"],
-    "users": ["password_hash"],
-}
-
-
-BOOLEAN_COLUMNS = {
-    "events": ["processed", "outlier"],
-    "rooms": ["is_public"],
-    "event_edges": ["is_state"],
-    "presence_list": ["accepted"],
-}
-
-
-APPEND_ONLY_TABLES = [
-    "event_content_hashes",
-    "event_reference_hashes",
-    "event_signatures",
-    "event_edge_hashes",
-    "events",
-    "event_json",
-    "state_events",
-    "room_memberships",
-    "feedback",
-    "topics",
-    "room_names",
-    "rooms",
-    "local_media_repository",
-    "local_media_repository_thumbnails",
-    "remote_media_cache",
-    "remote_media_cache_thumbnails",
-    "redactions",
-    "event_edges",
-    "event_auth",
-    "received_transactions",
-    "sent_transactions",
-    "transaction_id_to_pdu",
-    "users",
-    "state_groups",
-    "state_groups_state",
-    "event_to_state_groups",
-    "rejections",
-]
-
-
-class Store(object):
-    def __init__(self, db_pool, engine):
-        self.db_pool = db_pool
-        self.database_engine = engine
-
-    _simple_insert_txn = SQLBaseStore.__dict__["_simple_insert_txn"]
-    _simple_insert = SQLBaseStore.__dict__["_simple_insert"]
-
-    _simple_select_onecol_txn = SQLBaseStore.__dict__["_simple_select_onecol_txn"]
-    _simple_select_onecol = SQLBaseStore.__dict__["_simple_select_onecol"]
-    _simple_select_one_onecol = SQLBaseStore.__dict__["_simple_select_one_onecol"]
-    _simple_select_one_onecol_txn = SQLBaseStore.__dict__["_simple_select_one_onecol_txn"]
-
-    _simple_update_one = SQLBaseStore.__dict__["_simple_update_one"]
-    _simple_update_one_txn = SQLBaseStore.__dict__["_simple_update_one_txn"]
-
-    _execute_and_decode = SQLBaseStore.__dict__["_execute_and_decode"]
-
-    def runInteraction(self, desc, func, *args, **kwargs):
-        def r(conn):
-            try:
-                i = 0
-                N = 5
-                while True:
-                    try:
-                        txn = conn.cursor()
-                        return func(
-                            LoggingTransaction(txn, desc, self.database_engine),
-                            *args, **kwargs
-                        )
-                    except self.database_engine.module.DatabaseError as e:
-                        if self.database_engine.is_deadlock(e):
-                            logger.warn("[TXN DEADLOCK] {%s} %d/%d", desc, i, N)
-                            if i < N:
-                                i += 1
-                                conn.rollback()
-                                continue
-                        raise
-            except Exception as e:
-                logger.debug("[TXN FAIL] {%s}", desc, e)
-                raise
-
-        return self.db_pool.runWithConnection(r)
-
-    def insert_many_txn(self, txn, table, headers, rows):
-        sql = "INSERT INTO %s (%s) VALUES (%s)" % (
-            table,
-            ", ".join(k for k in headers),
-            ", ".join("%s" for _ in headers)
-        )
-
-        try:
-            txn.executemany(sql, rows)
-        except:
-            logger.exception(
-                "Failed to insert: %s",
-                table,
-            )
-            raise
-
-
-
-def chunks(n):
-    for i in itertools.count(0, n):
-        yield range(i, i+n)
-
-
-@defer.inlineCallbacks
-def handle_table(table, sqlite_store, mysql_store):
-    if table in APPEND_ONLY_TABLES:
-        # It's safe to just carry on inserting.
-        next_chunk = yield mysql_store._simple_select_one_onecol(
-            table="port_from_sqlite3",
-            keyvalues={"table_name": table},
-            retcol="rowid",
-            allow_none=True,
-        )
-
-        if next_chunk is None:
-            yield mysql_store._simple_insert(
-                table="port_from_sqlite3",
-                values={"table_name": table, "rowid": 0}
-            )
-
-            next_chunk = 0
-    else:
-        def delete_all(txn):
-            txn.execute(
-                "DELETE FROM port_from_sqlite3 WHERE table_name = %s",
-                (table,)
-            )
-            txn.execute("TRUNCATE %s CASCADE" % (table,))
-            mysql_store._simple_insert_txn(
-                txn,
-                table="port_from_sqlite3",
-                values={"table_name": table, "rowid": 0}
-            )
-
-        yield mysql_store.runInteraction(
-            "delete_non_append_only", delete_all
-        )
-
-        next_chunk = 0
-
-    logger.info("next_chunk for %s: %d", table, next_chunk)
-
-    N = 5000
-
-    select = "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
-
-    uni_col_names = UNICODE_COLUMNS.get(table, [])
-    bool_col_names = BOOLEAN_COLUMNS.get(table, [])
-    bin_col_names = BINARY_COLUMNS.get(table, [])
-
-    while True:
-        def r(txn):
-            txn.execute(select, (next_chunk, N,))
-            rows = txn.fetchall()
-            headers = [column[0] for column in txn.description]
-
-            return headers, rows
-
-        headers, rows = yield sqlite_store.runInteraction("select", r)
-
-        logger.info("Got %d rows for %s", len(rows), table)
-
-        if rows:
-            uni_cols = [i for i, h in enumerate(headers) if h in uni_col_names]
-            bool_cols = [i for i, h in enumerate(headers) if h in bool_col_names]
-            bin_cols = [i for i, h in enumerate(headers) if h in bin_col_names]
-            next_chunk = rows[-1][0] + 1
-
-            def conv(j, col):
-                if j in uni_cols:
-                    col = sqlite_store.database_engine.load_unicode(col)
-                if j in bool_cols:
-                    return bool(col)
-
-                if j in bin_cols:
-                    if isinstance(col, types.UnicodeType):
-                        col = buffer(col.encode("utf8"))
-
-                return col
-
-            for i, row in enumerate(rows):
-                rows[i] = tuple(
-                    mysql_store.database_engine.encode_parameter(
-                        conv(j, col)
-                    )
-                    for j, col in enumerate(row)
-                    if j > 0
-                )
-
-            def ins(txn):
-                mysql_store.insert_many_txn(txn, table, headers[1:], rows)
-
-                mysql_store._simple_update_one_txn(
-                    txn,
-                    table="port_from_sqlite3",
-                    keyvalues={"table_name": table},
-                    updatevalues={"rowid": next_chunk},
-                )
-
-            yield mysql_store.runInteraction("insert_many", ins)
-        else:
-            return
-
-
-def setup_db(db_config, database_engine):
-    db_conn = database_engine.module.connect(
-        **{
-            k: v for k, v in db_config.get("args", {}).items()
-            if not k.startswith("cp_")
-        }
-    )
-
-    database_engine.prepare_database(db_conn)
-
-    db_conn.commit()
-
-
-@defer.inlineCallbacks
-def main(sqlite_config, mysql_config):
-    try:
-        sqlite_db_pool = adbapi.ConnectionPool(
-            sqlite_config["name"],
-            **sqlite_config["args"]
-        )
-
-        mysql_db_pool = adbapi.ConnectionPool(
-            mysql_config["name"],
-            **mysql_config["args"]
-        )
-
-        sqlite_engine = create_engine("sqlite3")
-        mysql_engine = create_engine("psycopg2")
-
-        sqlite_store = Store(sqlite_db_pool, sqlite_engine)
-        mysql_store = Store(mysql_db_pool, mysql_engine)
-
-        # Step 1. Set up mysql database.
-        logger.info("Preparing sqlite database...")
-        setup_db(sqlite_config, sqlite_engine)
-
-        logger.info("Preparing mysql database...")
-        setup_db(mysql_config, mysql_engine)
-
-        # Step 2. Get tables.
-        logger.info("Fetching tables...")
-        tables = yield sqlite_store._simple_select_onecol(
-            table="sqlite_master",
-            keyvalues={
-                "type": "table",
-            },
-            retcol="name",
-        )
-
-        logger.info("Found %d tables", len(tables))
-
-        def create_port_table(txn):
-            txn.execute(
-                "CREATE TABLE port_from_sqlite3 ("
-                " table_name varchar(100) NOT NULL UNIQUE,"
-                " rowid bigint NOT NULL"
-                ")"
-            )
-
-        try:
-            yield mysql_store.runInteraction(
-                "create_port_table", create_port_table
-            )
-        except Exception as e:
-            logger.info("Failed to create port table: %s", e)
-
-        # Process tables.
-        yield defer.gatherResults(
-            [
-                handle_table(table, sqlite_store, mysql_store)
-                for table in tables
-                if table not in ["schema_version", "applied_schema_deltas"]
-                and not table.startswith("sqlite_")
-            ],
-            consumeErrors=True,
-        )
-
-        # for table in ["current_state_events"]:  # tables:
-        #     if table not in ["schema_version", "applied_schema_deltas"]:
-        #         if not table.startswith("sqlite_"):
-        #             yield handle_table(table, sqlite_store, mysql_store)
-    except:
-        logger.exception("")
-    finally:
-        reactor.stop()
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--sqlite-database")
-    parser.add_argument(
-        "--mysql-config", type=argparse.FileType('r'),
-    )
-
-    args = parser.parse_args()
-    logging.basicConfig(level=logging.INFO)
-
-    sqlite_config = {
-        "name": "sqlite3",
-        "args": {
-            "database": args.sqlite_database,
-            "cp_min": 1,
-            "cp_max": 1,
-            "check_same_thread": False,
-        },
-    }
-
-    mysql_config = yaml.safe_load(args.mysql_config)
-    # mysql_config["args"].update({
-    #     "sql_mode": "TRADITIONAL",
-    #     "charset": "utf8mb4",
-    #     "use_unicode": True,
-    #     "collation": "utf8mb4_bin",
-    # })
-
-    reactor.callWhenRunning(
-        main,
-        sqlite_config=sqlite_config,
-        mysql_config=mysql_config,
-    )
-
-    reactor.run()
diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py
index f8a33120b5..93500dd791 100755
--- a/synapse/app/homeserver.py
+++ b/synapse/app/homeserver.py
@@ -366,14 +366,7 @@ def setup(config_options):
     }
 
     name = db_config.get("name", None)
-    if name in ["MySQLdb", "mysql.connector"]:
-        db_config.setdefault("args", {}).update({
-            "sql_mode": "TRADITIONAL",
-            "charset": "utf8mb4",
-            "use_unicode": True,
-            "collation": "utf8mb4_bin",
-        })
-    elif name == "psycopg2":
+    if name == "psycopg2":
         pass
     elif name == "sqlite3":
         db_config.setdefault("args", {}).update({
diff --git a/synapse/storage/engines/__init__.py b/synapse/storage/engines/__init__.py
index 548d4e1b42..eb76df7f01 100644
--- a/synapse/storage/engines/__init__.py
+++ b/synapse/storage/engines/__init__.py
@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .maria import MariaEngine
 from .postgres import PostgresEngine
 from .sqlite3 import Sqlite3Engine
 
@@ -22,7 +21,6 @@ import importlib
 
 SUPPORTED_MODULE = {
     "sqlite3": Sqlite3Engine,
-    "mysql.connector": MariaEngine,
     "psycopg2": PostgresEngine,
 }
 
diff --git a/synapse/storage/engines/maria.py b/synapse/storage/engines/maria.py
deleted file mode 100644
index 90165f6849..0000000000
--- a/synapse/storage/engines/maria.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2015 OpenMarket Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.storage import prepare_database
-
-import types
-
-
-class MariaEngine(object):
-    def __init__(self, database_module):
-        self.module = database_module
-
-    def convert_param_style(self, sql):
-        return sql.replace("?", "%s")
-
-    def encode_parameter(self, param):
-        if isinstance(param, types.BufferType):
-            return bytes(param)
-        return param
-
-    def on_new_connection(self, db_conn):
-        pass
-
-    def prepare_database(self, db_conn):
-        cur = db_conn.cursor()
-        cur.execute(
-            "ALTER DATABASE CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"
-        )
-        db_conn.commit()
-        prepare_database(db_conn, self)
-
-    def is_deadlock(self, error):
-        if isinstance(error, self.module.DatabaseError):
-            return error.sqlstate == "40001" and error.errno == 1213
-        return False
-
-    def load_unicode(self, v):
-        return bytes(v).decode("UTF8")
diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql
index f9fbb6b448..3604ea8427 100644
--- a/synapse/storage/schema/delta/16/unique_constraints.sql
+++ b/synapse/storage/schema/delta/16/unique_constraints.sql
@@ -1,5 +1,5 @@
 
--- We can use SQLite features here, since mysql support was only added in v16
+-- We can use SQLite features here, since other db support was only added in v16
 
 --
 DELETE FROM current_state_events WHERE rowid not in (
-- 
cgit 1.5.1


From b8092fbc82edb3c7d8aa09f0756fa853ad6a6ad8 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 16 Apr 2015 11:17:52 +0100
Subject: Go back to storing JSON in TEXT

---
 synapse/storage/_base.py                                 |  3 ---
 synapse/storage/appservice.py                            |  4 +---
 synapse/storage/engines/postgres.py                      |  3 ---
 synapse/storage/engines/sqlite3.py                       |  5 -----
 synapse/storage/events.py                                | 16 ++++++++--------
 synapse/storage/profile.py                               |  5 +----
 synapse/storage/registration.py                          |  5 -----
 synapse/storage/schema/delta/15/appservice_txns.sql      |  2 +-
 synapse/storage/schema/full_schemas/11/im.sql            |  8 ++++----
 .../schema/full_schemas/16/application_services.sql      |  2 +-
 synapse/storage/schema/full_schemas/16/im.sql            |  8 ++++----
 11 files changed, 20 insertions(+), 41 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py
index 5ec1d2613e..f5952d1fc0 100644
--- a/synapse/storage/_base.py
+++ b/synapse/storage/_base.py
@@ -818,9 +818,6 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted, rejected_reason = res
 
-        internal_metadata = self.database_engine.load_unicode(internal_metadata)
-        js = self.database_engine.load_unicode(js)
-
         start_time = update_counter("select_event", start_time)
 
         result = self._get_event_from_row_txn(
diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py
index 40e05b3635..63d1af4e86 100644
--- a/synapse/storage/appservice.py
+++ b/synapse/storage/appservice.py
@@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
         new_txn_id = max(highest_txn_id, last_txn_id) + 1
 
         # Insert new txn into txn table
-        event_ids = buffer(
-            json.dumps([e.event_id for e in events]).encode("utf8")
-        )
+        event_ids = json.dumps([e.event_id for e in events])
         txn.execute(
             "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
             "VALUES(?,?,?)",
diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py
index 457c1f70a5..6f75245fa7 100644
--- a/synapse/storage/engines/postgres.py
+++ b/synapse/storage/engines/postgres.py
@@ -39,6 +39,3 @@ class PostgresEngine(object):
         if isinstance(error, self.module.DatabaseError):
             return error.pgcode in ["40001", "40P01"]
         return False
-
-    def load_unicode(self, v):
-        return bytes(v).decode("UTF8")
diff --git a/synapse/storage/engines/sqlite3.py b/synapse/storage/engines/sqlite3.py
index 389df35eb5..dd0d8e0e0f 100644
--- a/synapse/storage/engines/sqlite3.py
+++ b/synapse/storage/engines/sqlite3.py
@@ -37,8 +37,3 @@ class Sqlite3Engine(object):
 
     def is_deadlock(self, error):
         return False
-
-    def load_unicode(self, v):
-        if isinstance(v, types.UnicodeType):
-            return v
-        return bytes(v).decode("UTF8")
diff --git a/synapse/storage/events.py b/synapse/storage/events.py
index 0373d152b2..7dbf7a396a 100644
--- a/synapse/storage/events.py
+++ b/synapse/storage/events.py
@@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore):
 
         metadata_json = encode_canonical_json(
             event.internal_metadata.get_dict()
-        )
+        ).decode("UTF-8")
 
         # If we have already persisted this event, we don't need to do any
         # more processing.
@@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore):
                 )
                 txn.execute(
                     sql,
-                    (buffer(metadata_json), event.event_id,)
+                    (metadata_json, event.event_id,)
                 )
 
                 sql = (
@@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore):
             values={
                 "event_id": event.event_id,
                 "room_id": event.room_id,
-                "internal_metadata": buffer(metadata_json),
-                "json": buffer(encode_canonical_json(event_dict)),
+                "internal_metadata": metadata_json,
+                "json": encode_canonical_json(event_dict).decode("UTF-8"),
             },
         )
 
-        content = buffer(encode_canonical_json(
+        content = encode_canonical_json(
             event.content
-        ))
+        ).decode("UTF-8")
 
         vals = {
             "topological_ordering": event.depth,
@@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore):
             ]
         }
 
-        vals["unrecognized_keys"] = buffer(encode_canonical_json(
+        vals["unrecognized_keys"] = encode_canonical_json(
             unrec
-        ))
+        ).decode("UTF-8")
 
         sql = (
             "INSERT INTO events"
diff --git a/synapse/storage/profile.py b/synapse/storage/profile.py
index e33963d0b4..047698aa13 100644
--- a/synapse/storage/profile.py
+++ b/synapse/storage/profile.py
@@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore):
             desc="get_profile_displayname",
         )
 
-        if name:
-            name = self.database_engine.load_unicode(name)
-
         defer.returnValue(name)
 
     def set_profile_displayname(self, user_localpart, new_displayname):
         return self._simple_update_one(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname.encode("utf8")},
+            updatevalues={"displayname": new_displayname},
             desc="set_profile_displayname",
         )
 
diff --git a/synapse/storage/registration.py b/synapse/storage/registration.py
index 8a63fe4691..2a5c5080e4 100644
--- a/synapse/storage/registration.py
+++ b/synapse/storage/registration.py
@@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore):
             allow_none=True,
         )
 
-        if user_info:
-            user_info["password_hash"] = self.database_engine.load_unicode(
-                user_info["password_hash"]
-            )
-
         defer.returnValue(user_info)
 
     @cached()
diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql
index ddea8fc693..1c3324f415 100644
--- a/synapse/storage/schema/delta/15/appservice_txns.sql
+++ b/synapse/storage/schema/delta/15/appservice_txns.sql
@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids LONGBLOB NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index e9e09214d7..addbec5885 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
-    unrecognized_keys bytea,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id);
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
-    json bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index f08c5bcf76..5d63d57d59 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids bytea NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 17e4c949b9..5b4b494484 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
-    unrecognized_keys bytea,
+    content TEXT NOT NULL,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events (
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
-    json bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
-- 
cgit 1.5.1


From 0ec346d942450ac4a29c74d115428fd856af94b2 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 17 Apr 2015 11:10:20 +0100
Subject: Add unique index to room_aliases, remove duplicates on upgrade.
 Convert some columns back to TEXT from bytea

---
 synapse/storage/schema/delta/16/room_alias_index.sql    | 3 +++
 synapse/storage/schema/delta/16/unique_constraints.sql  | 8 ++++++++
 synapse/storage/schema/full_schemas/11/transactions.sql | 2 +-
 synapse/storage/schema/full_schemas/11/users.sql        | 2 +-
 synapse/storage/schema/full_schemas/16/room_aliases.sql | 4 ++++
 synapse/storage/schema/full_schemas/16/transactions.sql | 2 +-
 synapse/storage/schema/full_schemas/16/users.sql        | 2 +-
 7 files changed, 19 insertions(+), 4 deletions(-)
 create mode 100644 synapse/storage/schema/delta/16/room_alias_index.sql

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/delta/16/room_alias_index.sql
new file mode 100644
index 0000000000..f82486132b
--- /dev/null
+++ b/synapse/storage/schema/delta/16/room_alias_index.sql
@@ -0,0 +1,3 @@
+
+CREATE INDEX IF NOT EXISTS room_aliases_id ON room_aliases(room_id);
+CREATE INDEX IF NOT EXISTS room_alias_servers_alias ON room_alias_servers(room_alias);
diff --git a/synapse/storage/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/delta/16/unique_constraints.sql
index 3604ea8427..fecf11118c 100644
--- a/synapse/storage/schema/delta/16/unique_constraints.sql
+++ b/synapse/storage/schema/delta/16/unique_constraints.sql
@@ -70,3 +70,11 @@ DROP INDEX IF EXISTS presence_list_observers;
 CREATE UNIQUE INDEX presence_list_observers ON presence_list(
     user_id, observed_user_id
 );
+
+--
+DELETE FROM room_aliases WHERE rowid not in (
+    SELECT MIN(rowid) FROM room_aliases GROUP BY room_alias
+);
+
+DROP INDEX IF EXISTS room_aliases_id;
+CREATE INDEX room_aliases_id ON room_aliases(room_id);
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index f01d9eba32..524a696920 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json bytea,
+    response_json TEXT,
     ts BIGINT
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 2edc13c3b7..48a6aecfe8 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     access_token VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
-    user_agent VARCHAR(150) NOT NULL,
+    user_agent TEXT NOT NULL,
     last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
 );
diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql
index 952cae35b7..2c0853a2a9 100644
--- a/synapse/storage/schema/full_schemas/16/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql
@@ -19,7 +19,11 @@ CREATE TABLE IF NOT EXISTS room_aliases(
     UNIQUE (room_alias)
 );
 
+CREATE INDEX room_aliases_id ON room_aliases(room_id);
+
 CREATE TABLE IF NOT EXISTS room_alias_servers(
     room_alias VARCHAR(150) NOT NULL,
     server VARCHAR(150) NOT NULL
 );
+
+CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index bc64064936..ed431bd3af 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS sent_transactions(
     transaction_id VARCHAR(150),
     destination VARCHAR(150),
     response_code INTEGER DEFAULT 0,
-    response_json bytea,
+    response_json TEXT,
     ts BIGINT
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
index 006b249fc0..033e3244b5 100644
--- a/synapse/storage/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS user_ips (
     access_token VARCHAR(150) NOT NULL,
     device_id VARCHAR(150),
     ip VARCHAR(150) NOT NULL,
-    user_agent VARCHAR(150) NOT NULL,
+    user_agent TEXT NOT NULL,
     last_seen BIGINT NOT NULL
 );
 
-- 
cgit 1.5.1


From 18f82477011ee9279bde8ad995d654c116d3f652 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Tue, 28 Apr 2015 12:41:33 +0100
Subject: Use TEXT instead of VARCHAR(n), since PostgreSQL treats them the same
 except for a limit

---
 synapse/storage/schema/delta/12/v12.sql            | 18 +++---
 synapse/storage/schema/delta/13/v13.sql            | 10 ++--
 synapse/storage/schema/delta/14/v14.sql            |  4 +-
 .../storage/schema/delta/15/appservice_txns.sql    |  4 +-
 synapse/storage/schema/delta/16/users.sql          | 20 +++----
 .../storage/schema/full_schemas/11/event_edges.sql | 34 ++++++------
 .../schema/full_schemas/11/event_signatures.sql    | 20 +++----
 synapse/storage/schema/full_schemas/11/im.sql      | 64 +++++++++++-----------
 synapse/storage/schema/full_schemas/11/keys.sql    | 12 ++--
 .../schema/full_schemas/11/media_repository.sql    | 34 ++++++------
 .../storage/schema/full_schemas/11/presence.sql    | 12 ++--
 .../storage/schema/full_schemas/11/profiles.sql    |  6 +-
 .../storage/schema/full_schemas/11/redactions.sql  |  4 +-
 .../schema/full_schemas/11/room_aliases.sql        |  8 +--
 synapse/storage/schema/full_schemas/11/state.sql   | 14 ++---
 .../schema/full_schemas/11/transactions.sql        | 16 +++---
 synapse/storage/schema/full_schemas/11/users.sql   | 18 +++---
 .../full_schemas/16/application_services.sql       | 14 ++---
 .../storage/schema/full_schemas/16/event_edges.sql | 34 ++++++------
 .../schema/full_schemas/16/event_signatures.sql    | 20 +++----
 synapse/storage/schema/full_schemas/16/im.sql      | 64 +++++++++++-----------
 synapse/storage/schema/full_schemas/16/keys.sql    | 12 ++--
 .../schema/full_schemas/16/media_repository.sql    | 34 ++++++------
 .../storage/schema/full_schemas/16/presence.sql    | 12 ++--
 .../storage/schema/full_schemas/16/profiles.sql    |  6 +-
 synapse/storage/schema/full_schemas/16/push.sql    | 22 ++++----
 .../storage/schema/full_schemas/16/redactions.sql  |  4 +-
 .../schema/full_schemas/16/room_aliases.sql        |  8 +--
 synapse/storage/schema/full_schemas/16/state.sql   | 14 ++---
 .../schema/full_schemas/16/transactions.sql        | 16 +++---
 synapse/storage/schema/full_schemas/16/users.sql   | 18 +++---
 synapse/storage/schema/schema_version.sql          |  2 +-
 32 files changed, 289 insertions(+), 289 deletions(-)

(limited to 'synapse/storage/schema')

diff --git a/synapse/storage/schema/delta/12/v12.sql b/synapse/storage/schema/delta/12/v12.sql
index a246943f5a..878c36260a 100644
--- a/synapse/storage/schema/delta/12/v12.sql
+++ b/synapse/storage/schema/delta/12/v12.sql
@@ -14,16 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS rejections(
-    event_id VARCHAR(150) NOT NULL,
-    reason VARCHAR(150) NOT NULL,
-    last_check VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    last_check TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
   app_id VARCHAR(64) NOT NULL,
@@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers (
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name VARCHAR(150) NOT NULL,
-  rule_id VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
+  rule_id TEXT NOT NULL,
   priority_class TINYINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
-  conditions VARCHAR(150) NOT NULL,
-  actions VARCHAR(150) NOT NULL,
+  conditions TEXT NOT NULL,
+  actions TEXT NOT NULL,
   UNIQUE(user_name, rule_id)
 );
 
 CREATE INDEX IF NOT EXISTS push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
-  user_id VARCHAR(150),
+  user_id TEXT,
   filter_id BIGINT UNSIGNED,
   filter_json LONGBLOB
 );
diff --git a/synapse/storage/schema/delta/13/v13.sql b/synapse/storage/schema/delta/13/v13.sql
index d1da2b48e2..3265924013 100644
--- a/synapse/storage/schema/delta/13/v13.sql
+++ b/synapse/storage/schema/delta/13/v13.sql
@@ -15,10 +15,10 @@
 
 CREATE TABLE IF NOT EXISTS application_services(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    url VARCHAR(150),
-    token VARCHAR(150),
-    hs_token VARCHAR(150),
-    sender VARCHAR(150),
+    url TEXT,
+    token TEXT,
+    hs_token TEXT,
+    sender TEXT,
     UNIQUE(token)
 );
 
@@ -26,6 +26,6 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
     as_id BIGINT UNSIGNED NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex VARCHAR(150),
+    regex TEXT,
     FOREIGN KEY(as_id) REFERENCES application_services(id)
 );
diff --git a/synapse/storage/schema/delta/14/v14.sql b/synapse/storage/schema/delta/14/v14.sql
index e5fedc5851..1d09ad7a15 100644
--- a/synapse/storage/schema/delta/14/v14.sql
+++ b/synapse/storage/schema/delta/14/v14.sql
@@ -14,8 +14,8 @@
  */
 CREATE TABLE IF NOT EXISTS push_rules_enable (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
-  user_name VARCHAR(150) NOT NULL,
-  rule_id VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
+  rule_id TEXT NOT NULL,
   enabled TINYINT,
   UNIQUE(user_name, rule_id)
 );
diff --git a/synapse/storage/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/delta/15/appservice_txns.sql
index 1c3324f415..db2e720393 100644
--- a/synapse/storage/schema/delta/15/appservice_txns.sql
+++ b/synapse/storage/schema/delta/15/appservice_txns.sql
@@ -14,13 +14,13 @@
  */
 
 CREATE TABLE IF NOT EXISTS application_services_state(
-    as_id VARCHAR(150) PRIMARY KEY,
+    as_id TEXT PRIMARY KEY,
     state VARCHAR(5),
     last_txn INTEGER
 );
 
 CREATE TABLE IF NOT EXISTS application_services_txns(
-    as_id VARCHAR(150) NOT NULL,
+    as_id TEXT NOT NULL,
     txn_id INTEGER NOT NULL,
     event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
diff --git a/synapse/storage/schema/delta/16/users.sql b/synapse/storage/schema/delta/16/users.sql
index db27bdca02..cd0709250d 100644
--- a/synapse/storage/schema/delta/16/users.sql
+++ b/synapse/storage/schema/delta/16/users.sql
@@ -2,9 +2,9 @@
 -- MUST BE DONE BEFORE REMOVING ID COLUMN FROM USERS TABLE BELOW
 CREATE TABLE IF NOT EXISTS new_access_tokens(
     id BIGINT UNSIGNED PRIMARY KEY,
-    user_id VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    token VARCHAR(150) NOT NULL,
+    user_id TEXT NOT NULL,
+    device_id TEXT,
+    token TEXT NOT NULL,
     last_used BIGINT UNSIGNED,
     UNIQUE(token)
 );
@@ -20,8 +20,8 @@ ALTER TABLE new_access_tokens RENAME TO access_tokens;
 
 -- Remove ID column from `users` table
 CREATE TABLE IF NOT EXISTS new_users(
-    name VARCHAR(150),
-    password_hash VARCHAR(150),
+    name TEXT,
+    password_hash TEXT,
     creation_ts BIGINT UNSIGNED,
     admin BOOL DEFAULT 0 NOT NULL,
     UNIQUE(name)
@@ -36,11 +36,11 @@ ALTER TABLE new_users RENAME TO users;
 
 -- Remove UNIQUE constraint from `user_ips` table
 CREATE TABLE IF NOT EXISTS new_user_ips (
-    user_id VARCHAR(150) NOT NULL,
-    access_token VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    ip VARCHAR(150) NOT NULL,
-    user_agent VARCHAR(150) NOT NULL,
+    user_id TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
+    user_agent TEXT NOT NULL,
     last_seen BIGINT UNSIGNED NOT NULL
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/event_edges.sql b/synapse/storage/schema/full_schemas/11/event_edges.sql
index 05d0874f0d..f7020f7793 100644
--- a/synapse/storage/schema/full_schemas/11/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/11/event_edges.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
-    event_id VARCHAR(150) NOT NULL,
-    prev_event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    prev_event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
@@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
-    room_id VARCHAR(150) NOT NULL,
+    room_id TEXT NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
 );
@@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
-    event_id VARCHAR(150) NOT NULL,
-    destination VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    destination TEXT NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
@@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
-    event_id VARCHAR(150) NOT NULL,
-    auth_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    auth_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/event_signatures.sql b/synapse/storage/schema/full_schemas/11/event_signatures.sql
index 4291827368..636b2d3353 100644
--- a/synapse/storage/schema/full_schemas/11/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/11/event_signatures.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, algorithm)
 );
@@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, algorithm)
 );
@@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id VARCHAR(150),
-    signature_name VARCHAR(150),
-    key_id VARCHAR(150),
+    event_id TEXT,
+    signature_name TEXT,
+    key_id TEXT,
     signature bytea,
     UNIQUE (event_id, signature_name, key_id)
 );
@@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id VARCHAR(150),
-    prev_event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    prev_event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
diff --git a/synapse/storage/schema/full_schemas/11/im.sql b/synapse/storage/schema/full_schemas/11/im.sql
index addbec5885..1901654ac2 100644
--- a/synapse/storage/schema/full_schemas/11/im.sql
+++ b/synapse/storage/schema/full_schemas/11/im.sql
@@ -16,9 +16,9 @@
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering INTEGER PRIMARY KEY AUTOINCREMENT,
     topological_ordering BIGINT NOT NULL,
-    event_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     content TEXT NOT NULL,
     unrecognized_keys TEXT,
     processed BOOL NOT NULL,
@@ -33,8 +33,8 @@ CREATE INDEX events_room_id ON events (room_id);
 
 
 CREATE TABLE IF NOT EXISTS event_json(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     internal_metadata TEXT NOT NULL,
     json TEXT NOT NULL,
     UNIQUE (event_id)
@@ -44,11 +44,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
-    prev_state VARCHAR(150),
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    prev_state TEXT,
     UNIQUE (event_id)
 );
 
@@ -58,10 +58,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
     UNIQUE (room_id, type, state_key)
 );
 
@@ -71,11 +71,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type);
 CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id VARCHAR(150) NOT NULL,
-    user_id VARCHAR(150) NOT NULL,
-    sender VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    membership VARCHAR(150) NOT NULL
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    membership TEXT NOT NULL
 );
 
 CREATE INDEX room_memberships_event_id ON room_memberships (event_id);
@@ -83,16 +83,16 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
-    event_id VARCHAR(150) NOT NULL,
-    feedback_type VARCHAR(150),
-    target_event_id VARCHAR(150),
-    sender VARCHAR(150),
-    room_id VARCHAR(150)
+    event_id TEXT NOT NULL,
+    feedback_type TEXT,
+    target_event_id TEXT,
+    sender TEXT,
+    room_id TEXT
 );
 
 CREATE TABLE IF NOT EXISTS topics(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     topic TEXT NOT NULL
 );
 
@@ -100,8 +100,8 @@ CREATE INDEX topics_event_id ON topics(event_id);
 CREATE INDEX topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     name TEXT NOT NULL
 );
 
@@ -109,14 +109,14 @@ CREATE INDEX room_names_event_id ON room_names(event_id);
 CREATE INDEX room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
-    room_id VARCHAR(150) PRIMARY KEY NOT NULL,
+    room_id TEXT PRIMARY KEY NOT NULL,
     is_public BOOL,
-    creator VARCHAR(150)
+    creator TEXT
 );
 
 CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id VARCHAR(150) NOT NULL,
-    host VARCHAR(150) NOT NULL,
+    room_id TEXT NOT NULL,
+    host TEXT NOT NULL,
     UNIQUE (room_id, host)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/keys.sql b/synapse/storage/schema/full_schemas/11/keys.sql
index 459b510427..afc142045e 100644
--- a/synapse/storage/schema/full_schemas/11/keys.sql
+++ b/synapse/storage/schema/full_schemas/11/keys.sql
@@ -13,18 +13,18 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name VARCHAR(150), -- Server name.
-  fingerprint VARCHAR(150), -- Certificate fingerprint.
-  from_server VARCHAR(150), -- Which key server the certificate was fetched from.
+  server_name TEXT, -- Server name.
+  fingerprint TEXT, -- Certificate fingerprint.
+  from_server TEXT, -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate bytea, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name VARCHAR(150), -- Server name.
-  key_id VARCHAR(150), -- Key version.
-  from_server VARCHAR(150), -- Which key server the key was fetched form.
+  server_name TEXT, -- Server name.
+  key_id TEXT, -- Key version.
+  from_server TEXT, -- Which key server the key was fetched form.
   ts_added_ms BIGINT, -- When the key was added.
   verify_key bytea, -- NACL verification key.
   UNIQUE (server_name, key_id)
diff --git a/synapse/storage/schema/full_schemas/11/media_repository.sql b/synapse/storage/schema/full_schemas/11/media_repository.sql
index 6e0ee0db41..e927e581d1 100644
--- a/synapse/storage/schema/full_schemas/11/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/11/media_repository.sql
@@ -14,21 +14,21 @@
  */
 
 CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id VARCHAR(150), -- The id used to refer to the media.
-    media_type VARCHAR(150), -- The MIME-type of the media.
+    media_id TEXT, -- The id used to refer to the media.
+    media_type TEXT, -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(150), -- The name the media was uploaded with.
-    user_id VARCHAR(150), -- The user who uploaded the file.
+    upload_name TEXT, -- The name the media was uploaded with.
+    user_id TEXT, -- The user who uploaded the file.
     UNIQUE (media_id)
 );
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_id TEXT, -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
-    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail.
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+    thumbnail_method TEXT, -- The method used to make the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
@@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin VARCHAR(150), -- The remote HS the media came from.
-    media_id VARCHAR(150), -- The id used to refer to the media on that server.
-    media_type VARCHAR(150), -- The MIME-type of the media.
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media on that server.
+    media_type TEXT, -- The MIME-type of the media.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    upload_name TEXT, -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    filesystem_id TEXT, -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
 );
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin VARCHAR(150), -- The remote HS the media came from.
-    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail
-    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_method TEXT, -- The method used to make the thumbnail
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    filesystem_id TEXT, -- The name used to store the media on disk.
     UNIQUE (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
diff --git a/synapse/storage/schema/full_schemas/11/presence.sql b/synapse/storage/schema/full_schemas/11/presence.sql
index fce324b890..d8d82e9fe3 100644
--- a/synapse/storage/schema/full_schemas/11/presence.sql
+++ b/synapse/storage/schema/full_schemas/11/presence.sql
@@ -13,23 +13,23 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS presence(
-  user_id VARCHAR(150) NOT NULL,
+  user_id TEXT NOT NULL,
   state VARCHAR(20),
-  status_msg VARCHAR(150),
+  status_msg TEXT,
   mtime BIGINT -- miliseconds since last state change
 );
 
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id VARCHAR(150) NOT NULL,
-  observer_user_id VARCHAR(150) NOT NULL -- a UserID,
+  observed_user_id TEXT NOT NULL,
+  observer_user_id TEXT NOT NULL -- a UserID,
 );
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
-  user_id VARCHAR(150) NOT NULL,
-  observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  user_id TEXT NOT NULL,
+  observed_user_id TEXT NOT NULL, -- a UserID,
   accepted BOOLEAN NOT NULL
 );
diff --git a/synapse/storage/schema/full_schemas/11/profiles.sql b/synapse/storage/schema/full_schemas/11/profiles.sql
index ffe75edf9f..26e4204437 100644
--- a/synapse/storage/schema/full_schemas/11/profiles.sql
+++ b/synapse/storage/schema/full_schemas/11/profiles.sql
@@ -13,7 +13,7 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS profiles(
-    user_id VARCHAR(150) NOT NULL,
-    displayname VARCHAR(150),
-    avatar_url VARCHAR(150)
+    user_id TEXT NOT NULL,
+    displayname TEXT,
+    avatar_url TEXT
 );
diff --git a/synapse/storage/schema/full_schemas/11/redactions.sql b/synapse/storage/schema/full_schemas/11/redactions.sql
index 492fd22033..69621955d4 100644
--- a/synapse/storage/schema/full_schemas/11/redactions.sql
+++ b/synapse/storage/schema/full_schemas/11/redactions.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS redactions (
-    event_id VARCHAR(150) NOT NULL,
-    redacts VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    redacts TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/11/room_aliases.sql b/synapse/storage/schema/full_schemas/11/room_aliases.sql
index 6226913227..5027b1e3f6 100644
--- a/synapse/storage/schema/full_schemas/11/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/11/room_aliases.sql
@@ -14,11 +14,11 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL
+    room_alias TEXT NOT NULL,
+    room_id TEXT NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias VARCHAR(150) NOT NULL,
-    server VARCHAR(150) NOT NULL
+    room_alias TEXT NOT NULL,
+    server TEXT NOT NULL
 );
diff --git a/synapse/storage/schema/full_schemas/11/state.sql b/synapse/storage/schema/full_schemas/11/state.sql
index 62c20819fd..ffd164ab71 100644
--- a/synapse/storage/schema/full_schemas/11/state.sql
+++ b/synapse/storage/schema/full_schemas/11/state.sql
@@ -15,20 +15,20 @@
 
 CREATE TABLE IF NOT EXISTS state_groups(
     id INTEGER PRIMARY KEY,
-    room_id VARCHAR(150) NOT NULL,
-    event_id VARCHAR(150) NOT NULL
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group INTEGER NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
-    event_id VARCHAR(150) NOT NULL
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    event_id TEXT NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
     state_group INTEGER NOT NULL,
     UNIQUE (event_id)
 );
diff --git a/synapse/storage/schema/full_schemas/11/transactions.sql b/synapse/storage/schema/full_schemas/11/transactions.sql
index 524a696920..cc5b54f5aa 100644
--- a/synapse/storage/schema/full_schemas/11/transactions.sql
+++ b/synapse/storage/schema/full_schemas/11/transactions.sql
@@ -14,8 +14,8 @@
  */
 -- Stores what transaction ids we have received and what our response was
 CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id VARCHAR(150),
-    origin VARCHAR(150),
+    transaction_id TEXT,
+    origin TEXT,
     ts BIGINT,
     response_code INTEGER,
     response_json bytea,
@@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
     id INTEGER PRIMARY KEY AUTOINCREMENT, -- This is used to apply insertion ordering
-    transaction_id VARCHAR(150),
-    destination VARCHAR(150),
+    transaction_id TEXT,
+    destination TEXT,
     response_code INTEGER DEFAULT 0,
     response_json TEXT,
     ts BIGINT
@@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
 -- For sent transactions only.
 CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
-    destination VARCHAR(150),
-    pdu_id VARCHAR(150),
-    pdu_origin VARCHAR(150)
+    destination TEXT,
+    pdu_id TEXT,
+    pdu_origin TEXT
 );
 
 CREATE INDEX transaction_id_to_pdu_tx ON transaction_id_to_pdu(transaction_id, destination);
@@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
-    destination VARCHAR(150) PRIMARY KEY,
+    destination TEXT PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/11/users.sql b/synapse/storage/schema/full_schemas/11/users.sql
index 48a6aecfe8..eec3da3c35 100644
--- a/synapse/storage/schema/full_schemas/11/users.sql
+++ b/synapse/storage/schema/full_schemas/11/users.sql
@@ -14,8 +14,8 @@
  */
 CREATE TABLE IF NOT EXISTS users(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name VARCHAR(150),
-    password_hash VARCHAR(150),
+    name TEXT,
+    password_hash TEXT,
     creation_ts BIGINT,
     admin SMALLINT DEFAULT 0 NOT NULL,
     UNIQUE(name)
@@ -23,18 +23,18 @@ CREATE TABLE IF NOT EXISTS users(
 
 CREATE TABLE IF NOT EXISTS access_tokens(
     id INTEGER PRIMARY KEY AUTOINCREMENT,
-    user_id VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    token VARCHAR(150) NOT NULL,
+    user_id TEXT NOT NULL,
+    device_id TEXT,
+    token TEXT NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
 );
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user VARCHAR(150) NOT NULL,
-    access_token VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    ip VARCHAR(150) NOT NULL,
+    user TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
     user_agent TEXT NOT NULL,
     last_seen BIGINT NOT NULL,
     UNIQUE (user, access_token, ip, user_agent)
diff --git a/synapse/storage/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/full_schemas/16/application_services.sql
index 5d63d57d59..d382d63fbd 100644
--- a/synapse/storage/schema/full_schemas/16/application_services.sql
+++ b/synapse/storage/schema/full_schemas/16/application_services.sql
@@ -15,10 +15,10 @@
 
 CREATE TABLE IF NOT EXISTS application_services(
     id BIGINT PRIMARY KEY,
-    url VARCHAR(150),
-    token VARCHAR(150),
-    hs_token VARCHAR(150),
-    sender VARCHAR(150),
+    url TEXT,
+    token TEXT,
+    hs_token TEXT,
+    sender TEXT,
     UNIQUE(token)
 );
 
@@ -26,18 +26,18 @@ CREATE TABLE IF NOT EXISTS application_services_regex(
     id BIGINT PRIMARY KEY,
     as_id BIGINT NOT NULL,
     namespace INTEGER,  /* enum[room_id|room_alias|user_id] */
-    regex VARCHAR(150),
+    regex TEXT,
     FOREIGN KEY(as_id) REFERENCES application_services(id)
 );
 
 CREATE TABLE IF NOT EXISTS application_services_state(
-    as_id VARCHAR(150) PRIMARY KEY,
+    as_id TEXT PRIMARY KEY,
     state VARCHAR(5),
     last_txn INTEGER
 );
 
 CREATE TABLE IF NOT EXISTS application_services_txns(
-    as_id VARCHAR(150) NOT NULL,
+    as_id TEXT NOT NULL,
     txn_id INTEGER NOT NULL,
     event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
diff --git a/synapse/storage/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/full_schemas/16/event_edges.sql
index 05d0874f0d..f7020f7793 100644
--- a/synapse/storage/schema/full_schemas/16/event_edges.sql
+++ b/synapse/storage/schema/full_schemas/16/event_edges.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_forward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -24,8 +24,8 @@ CREATE INDEX ev_extrem_id ON event_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_backward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -34,9 +34,9 @@ CREATE INDEX ev_b_extrem_id ON event_backward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edges(
-    event_id VARCHAR(150) NOT NULL,
-    prev_event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    prev_event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     is_state BOOL NOT NULL,
     UNIQUE (event_id, prev_event_id, room_id, is_state)
 );
@@ -46,7 +46,7 @@ CREATE INDEX ev_edges_prev_id ON event_edges(prev_event_id);
 
 
 CREATE TABLE IF NOT EXISTS room_depth(
-    room_id VARCHAR(150) NOT NULL,
+    room_id TEXT NOT NULL,
     min_depth INTEGER NOT NULL,
     UNIQUE (room_id)
 );
@@ -55,8 +55,8 @@ CREATE INDEX room_depth_room ON room_depth(room_id);
 
 
 create TABLE IF NOT EXISTS event_destinations(
-    event_id VARCHAR(150) NOT NULL,
-    destination VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    destination TEXT NOT NULL,
     delivered_ts BIGINT DEFAULT 0, -- or 0 if not delivered
     UNIQUE (event_id, destination)
 );
@@ -65,10 +65,10 @@ CREATE INDEX event_destinations_id ON event_destinations(event_id);
 
 
 CREATE TABLE IF NOT EXISTS state_forward_extremities(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
     UNIQUE (event_id, room_id)
 );
 
@@ -79,9 +79,9 @@ CREATE INDEX st_extrem_id ON state_forward_extremities(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_auth(
-    event_id VARCHAR(150) NOT NULL,
-    auth_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    auth_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (event_id, auth_id, room_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/full_schemas/16/event_signatures.sql
index 4291827368..636b2d3353 100644
--- a/synapse/storage/schema/full_schemas/16/event_signatures.sql
+++ b/synapse/storage/schema/full_schemas/16/event_signatures.sql
@@ -14,8 +14,8 @@
  */
 
 CREATE TABLE IF NOT EXISTS event_content_hashes (
-    event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, algorithm)
 );
@@ -24,8 +24,8 @@ CREATE INDEX event_content_hashes_id ON event_content_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_reference_hashes (
-    event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, algorithm)
 );
@@ -34,9 +34,9 @@ CREATE INDEX event_reference_hashes_id ON event_reference_hashes(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_signatures (
-    event_id VARCHAR(150),
-    signature_name VARCHAR(150),
-    key_id VARCHAR(150),
+    event_id TEXT,
+    signature_name TEXT,
+    key_id TEXT,
     signature bytea,
     UNIQUE (event_id, signature_name, key_id)
 );
@@ -45,9 +45,9 @@ CREATE INDEX event_signatures_id ON event_signatures(event_id);
 
 
 CREATE TABLE IF NOT EXISTS event_edge_hashes(
-    event_id VARCHAR(150),
-    prev_event_id VARCHAR(150),
-    algorithm VARCHAR(150),
+    event_id TEXT,
+    prev_event_id TEXT,
+    algorithm TEXT,
     hash bytea,
     UNIQUE (event_id, prev_event_id, algorithm)
 );
diff --git a/synapse/storage/schema/full_schemas/16/im.sql b/synapse/storage/schema/full_schemas/16/im.sql
index 5b4b494484..576653a3c9 100644
--- a/synapse/storage/schema/full_schemas/16/im.sql
+++ b/synapse/storage/schema/full_schemas/16/im.sql
@@ -16,9 +16,9 @@
 CREATE TABLE IF NOT EXISTS events(
     stream_ordering INTEGER PRIMARY KEY,
     topological_ordering BIGINT NOT NULL,
-    event_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     content TEXT NOT NULL,
     unrecognized_keys TEXT,
     processed BOOL NOT NULL,
@@ -37,8 +37,8 @@ CREATE INDEX events_order_room ON events (
 
 
 CREATE TABLE IF NOT EXISTS event_json(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     internal_metadata TEXT NOT NULL,
     json TEXT NOT NULL,
     UNIQUE (event_id)
@@ -48,11 +48,11 @@ CREATE INDEX event_json_room_id ON event_json(room_id);
 
 
 CREATE TABLE IF NOT EXISTS state_events(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
-    prev_state VARCHAR(150),
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    prev_state TEXT,
     UNIQUE (event_id)
 );
 
@@ -62,10 +62,10 @@ CREATE INDEX state_events_state_key ON state_events (state_key);
 
 
 CREATE TABLE IF NOT EXISTS current_state_events(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
     UNIQUE (event_id),
     UNIQUE (room_id, type, state_key)
 );
@@ -75,11 +75,11 @@ CREATE INDEX current_state_events_type ON current_state_events (type);
 CREATE INDEX current_state_events_state_key ON current_state_events (state_key);
 
 CREATE TABLE IF NOT EXISTS room_memberships(
-    event_id VARCHAR(150) NOT NULL,
-    user_id VARCHAR(150) NOT NULL,
-    sender VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    membership VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    user_id TEXT NOT NULL,
+    sender TEXT NOT NULL,
+    room_id TEXT NOT NULL,
+    membership TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
@@ -87,17 +87,17 @@ CREATE INDEX room_memberships_room_id ON room_memberships (room_id);
 CREATE INDEX room_memberships_user_id ON room_memberships (user_id);
 
 CREATE TABLE IF NOT EXISTS feedback(
-    event_id VARCHAR(150) NOT NULL,
-    feedback_type VARCHAR(150),
-    target_event_id VARCHAR(150),
-    sender VARCHAR(150),
-    room_id VARCHAR(150),
+    event_id TEXT NOT NULL,
+    feedback_type TEXT,
+    target_event_id TEXT,
+    sender TEXT,
+    room_id TEXT,
     UNIQUE (event_id)
 );
 
 CREATE TABLE IF NOT EXISTS topics(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     topic TEXT NOT NULL,
     UNIQUE (event_id)
 );
@@ -105,8 +105,8 @@ CREATE TABLE IF NOT EXISTS topics(
 CREATE INDEX topics_room_id ON topics(room_id);
 
 CREATE TABLE IF NOT EXISTS room_names(
-    event_id VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     name TEXT NOT NULL,
     UNIQUE (event_id)
 );
@@ -114,14 +114,14 @@ CREATE TABLE IF NOT EXISTS room_names(
 CREATE INDEX room_names_room_id ON room_names(room_id);
 
 CREATE TABLE IF NOT EXISTS rooms(
-    room_id VARCHAR(150) PRIMARY KEY NOT NULL,
+    room_id TEXT PRIMARY KEY NOT NULL,
     is_public BOOL,
-    creator VARCHAR(150)
+    creator TEXT
 );
 
 CREATE TABLE IF NOT EXISTS room_hosts(
-    room_id VARCHAR(150) NOT NULL,
-    host VARCHAR(150) NOT NULL,
+    room_id TEXT NOT NULL,
+    host TEXT NOT NULL,
     UNIQUE (room_id, host)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index 459b510427..afc142045e 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -13,18 +13,18 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name VARCHAR(150), -- Server name.
-  fingerprint VARCHAR(150), -- Certificate fingerprint.
-  from_server VARCHAR(150), -- Which key server the certificate was fetched from.
+  server_name TEXT, -- Server name.
+  fingerprint TEXT, -- Certificate fingerprint.
+  from_server TEXT, -- Which key server the certificate was fetched from.
   ts_added_ms BIGINT, -- When the certifcate was added.
   tls_certificate bytea, -- DER encoded x509 certificate.
   UNIQUE (server_name, fingerprint)
 );
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
-  server_name VARCHAR(150), -- Server name.
-  key_id VARCHAR(150), -- Key version.
-  from_server VARCHAR(150), -- Which key server the key was fetched form.
+  server_name TEXT, -- Server name.
+  key_id TEXT, -- Key version.
+  from_server TEXT, -- Which key server the key was fetched form.
   ts_added_ms BIGINT, -- When the key was added.
   verify_key bytea, -- NACL verification key.
   UNIQUE (server_name, key_id)
diff --git a/synapse/storage/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/full_schemas/16/media_repository.sql
index 0e819fca38..dacbda40ca 100644
--- a/synapse/storage/schema/full_schemas/16/media_repository.sql
+++ b/synapse/storage/schema/full_schemas/16/media_repository.sql
@@ -14,21 +14,21 @@
  */
 
 CREATE TABLE IF NOT EXISTS local_media_repository (
-    media_id VARCHAR(150), -- The id used to refer to the media.
-    media_type VARCHAR(150), -- The MIME-type of the media.
+    media_id TEXT, -- The id used to refer to the media.
+    media_type TEXT, -- The MIME-type of the media.
     media_length INTEGER, -- Length of the media in bytes.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(150), -- The name the media was uploaded with.
-    user_id VARCHAR(150), -- The user who uploaded the file.
+    upload_name TEXT, -- The name the media was uploaded with.
+    user_id TEXT, -- The user who uploaded the file.
     UNIQUE (media_id)
 );
 
 CREATE TABLE IF NOT EXISTS local_media_repository_thumbnails (
-    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_id TEXT, -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
-    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail.
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
+    thumbnail_method TEXT, -- The method used to make the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
     UNIQUE (
         media_id, thumbnail_width, thumbnail_height, thumbnail_type
@@ -39,25 +39,25 @@ CREATE INDEX local_media_repository_thumbnails_media_id
     ON local_media_repository_thumbnails (media_id);
 
 CREATE TABLE IF NOT EXISTS remote_media_cache (
-    media_origin VARCHAR(150), -- The remote HS the media came from.
-    media_id VARCHAR(150), -- The id used to refer to the media on that server.
-    media_type VARCHAR(150), -- The MIME-type of the media.
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media on that server.
+    media_type TEXT, -- The MIME-type of the media.
     created_ts BIGINT, -- When the content was uploaded in ms.
-    upload_name VARCHAR(150), -- The name the media was uploaded with.
+    upload_name TEXT, -- The name the media was uploaded with.
     media_length INTEGER, -- Length of the media in bytes.
-    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    filesystem_id TEXT, -- The name used to store the media on disk.
     UNIQUE (media_origin, media_id)
 );
 
 CREATE TABLE IF NOT EXISTS remote_media_cache_thumbnails (
-    media_origin VARCHAR(150), -- The remote HS the media came from.
-    media_id VARCHAR(150), -- The id used to refer to the media.
+    media_origin TEXT, -- The remote HS the media came from.
+    media_id TEXT, -- The id used to refer to the media.
     thumbnail_width INTEGER, -- The width of the thumbnail in pixels.
     thumbnail_height INTEGER, -- The height of the thumbnail in pixels.
-    thumbnail_method VARCHAR(150), -- The method used to make the thumbnail
-    thumbnail_type VARCHAR(150), -- The MIME-type of the thumbnail.
+    thumbnail_method TEXT, -- The method used to make the thumbnail
+    thumbnail_type TEXT, -- The MIME-type of the thumbnail.
     thumbnail_length INTEGER, -- The length of the thumbnail in bytes.
-    filesystem_id VARCHAR(150), -- The name used to store the media on disk.
+    filesystem_id TEXT, -- The name used to store the media on disk.
     UNIQUE (
         media_origin, media_id, thumbnail_width, thumbnail_height,
         thumbnail_type
diff --git a/synapse/storage/schema/full_schemas/16/presence.sql b/synapse/storage/schema/full_schemas/16/presence.sql
index 9c41be296e..80088413ba 100644
--- a/synapse/storage/schema/full_schemas/16/presence.sql
+++ b/synapse/storage/schema/full_schemas/16/presence.sql
@@ -13,9 +13,9 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS presence(
-  user_id VARCHAR(150) NOT NULL,
+  user_id TEXT NOT NULL,
   state VARCHAR(20),
-  status_msg VARCHAR(150),
+  status_msg TEXT,
   mtime BIGINT, -- miliseconds since last state change
   UNIQUE (user_id)
 );
@@ -23,16 +23,16 @@ CREATE TABLE IF NOT EXISTS presence(
 -- For each of /my/ users which possibly-remote users are allowed to see their
 -- presence state
 CREATE TABLE IF NOT EXISTS presence_allow_inbound(
-  observed_user_id VARCHAR(150) NOT NULL,
-  observer_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  observed_user_id TEXT NOT NULL,
+  observer_user_id TEXT NOT NULL, -- a UserID,
   UNIQUE (observed_user_id, observer_user_id)
 );
 
 -- For each of /my/ users (watcher), which possibly-remote users are they
 -- watching?
 CREATE TABLE IF NOT EXISTS presence_list(
-  user_id VARCHAR(150) NOT NULL,
-  observed_user_id VARCHAR(150) NOT NULL, -- a UserID,
+  user_id TEXT NOT NULL,
+  observed_user_id TEXT NOT NULL, -- a UserID,
   accepted BOOLEAN NOT NULL,
   UNIQUE (user_id, observed_user_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/full_schemas/16/profiles.sql
index 21c58a99bc..934be86520 100644
--- a/synapse/storage/schema/full_schemas/16/profiles.sql
+++ b/synapse/storage/schema/full_schemas/16/profiles.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS profiles(
-    user_id VARCHAR(150) NOT NULL,
-    displayname VARCHAR(150),
-    avatar_url VARCHAR(150),
+    user_id TEXT NOT NULL,
+    displayname TEXT,
+    avatar_url TEXT,
     UNIQUE(user_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/push.sql b/synapse/storage/schema/full_schemas/16/push.sql
index 5c0c7bc201..db6e05cbdf 100644
--- a/synapse/storage/schema/full_schemas/16/push.sql
+++ b/synapse/storage/schema/full_schemas/16/push.sql
@@ -14,16 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS rejections(
-    event_id VARCHAR(150) NOT NULL,
-    reason VARCHAR(150) NOT NULL,
-    last_check VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    last_check TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
 -- Push notification endpoints that users have configured
 CREATE TABLE IF NOT EXISTS pushers (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
   profile_tag VARCHAR(32) NOT NULL,
   kind VARCHAR(8) NOT NULL,
   app_id VARCHAR(64) NOT NULL,
@@ -41,19 +41,19 @@ CREATE TABLE IF NOT EXISTS pushers (
 
 CREATE TABLE IF NOT EXISTS push_rules (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(150) NOT NULL,
-  rule_id VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
+  rule_id TEXT NOT NULL,
   priority_class SMALLINT NOT NULL,
   priority INTEGER NOT NULL DEFAULT 0,
-  conditions VARCHAR(150) NOT NULL,
-  actions VARCHAR(150) NOT NULL,
+  conditions TEXT NOT NULL,
+  actions TEXT NOT NULL,
   UNIQUE(user_name, rule_id)
 );
 
 CREATE INDEX push_rules_user_name on push_rules (user_name);
 
 CREATE TABLE IF NOT EXISTS user_filters(
-  user_id VARCHAR(150),
+  user_id TEXT,
   filter_id BIGINT,
   filter_json bytea
 );
@@ -64,8 +64,8 @@ CREATE INDEX user_filters_by_user_id_filter_id ON user_filters(
 
 CREATE TABLE IF NOT EXISTS push_rules_enable (
   id BIGINT PRIMARY KEY,
-  user_name VARCHAR(150) NOT NULL,
-  rule_id VARCHAR(150) NOT NULL,
+  user_name TEXT NOT NULL,
+  rule_id TEXT NOT NULL,
   enabled SMALLINT,
   UNIQUE(user_name, rule_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/full_schemas/16/redactions.sql
index 492fd22033..69621955d4 100644
--- a/synapse/storage/schema/full_schemas/16/redactions.sql
+++ b/synapse/storage/schema/full_schemas/16/redactions.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS redactions (
-    event_id VARCHAR(150) NOT NULL,
-    redacts VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
+    redacts TEXT NOT NULL,
     UNIQUE (event_id)
 );
 
diff --git a/synapse/storage/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/full_schemas/16/room_aliases.sql
index 2c0853a2a9..412bb97fad 100644
--- a/synapse/storage/schema/full_schemas/16/room_aliases.sql
+++ b/synapse/storage/schema/full_schemas/16/room_aliases.sql
@@ -14,16 +14,16 @@
  */
 
 CREATE TABLE IF NOT EXISTS room_aliases(
-    room_alias VARCHAR(150) NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
+    room_alias TEXT NOT NULL,
+    room_id TEXT NOT NULL,
     UNIQUE (room_alias)
 );
 
 CREATE INDEX room_aliases_id ON room_aliases(room_id);
 
 CREATE TABLE IF NOT EXISTS room_alias_servers(
-    room_alias VARCHAR(150) NOT NULL,
-    server VARCHAR(150) NOT NULL
+    room_alias TEXT NOT NULL,
+    server TEXT NOT NULL
 );
 
 CREATE INDEX room_alias_servers_alias ON room_alias_servers(room_alias);
diff --git a/synapse/storage/schema/full_schemas/16/state.sql b/synapse/storage/schema/full_schemas/16/state.sql
index b0cd5ee75a..705cac6ce9 100644
--- a/synapse/storage/schema/full_schemas/16/state.sql
+++ b/synapse/storage/schema/full_schemas/16/state.sql
@@ -15,20 +15,20 @@
 
 CREATE TABLE IF NOT EXISTS state_groups(
     id BIGINT PRIMARY KEY,
-    room_id VARCHAR(150) NOT NULL,
-    event_id VARCHAR(150) NOT NULL
+    room_id TEXT NOT NULL,
+    event_id TEXT NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS state_groups_state(
     state_group BIGINT NOT NULL,
-    room_id VARCHAR(150) NOT NULL,
-    type VARCHAR(150) NOT NULL,
-    state_key VARCHAR(150) NOT NULL,
-    event_id VARCHAR(150) NOT NULL
+    room_id TEXT NOT NULL,
+    type TEXT NOT NULL,
+    state_key TEXT NOT NULL,
+    event_id TEXT NOT NULL
 );
 
 CREATE TABLE IF NOT EXISTS event_to_state_groups(
-    event_id VARCHAR(150) NOT NULL,
+    event_id TEXT NOT NULL,
     state_group BIGINT NOT NULL,
     UNIQUE (event_id)
 );
diff --git a/synapse/storage/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/full_schemas/16/transactions.sql
index ed431bd3af..1ab77cdb63 100644
--- a/synapse/storage/schema/full_schemas/16/transactions.sql
+++ b/synapse/storage/schema/full_schemas/16/transactions.sql
@@ -14,8 +14,8 @@
  */
 -- Stores what transaction ids we have received and what our response was
 CREATE TABLE IF NOT EXISTS received_transactions(
-    transaction_id VARCHAR(150),
-    origin VARCHAR(150),
+    transaction_id TEXT,
+    origin TEXT,
     ts BIGINT,
     response_code INTEGER,
     response_json bytea,
@@ -30,8 +30,8 @@ CREATE INDEX transactions_have_ref ON received_transactions(origin, has_been_ref
 -- since referenced the transaction in another outgoing transaction
 CREATE TABLE IF NOT EXISTS sent_transactions(
     id BIGINT PRIMARY KEY, -- This is used to apply insertion ordering
-    transaction_id VARCHAR(150),
-    destination VARCHAR(150),
+    transaction_id TEXT,
+    destination TEXT,
     response_code INTEGER DEFAULT 0,
     response_json TEXT,
     ts BIGINT
@@ -47,9 +47,9 @@ CREATE INDEX sent_transaction_sent ON sent_transactions(response_code);
 -- For sent transactions only.
 CREATE TABLE IF NOT EXISTS transaction_id_to_pdu(
     transaction_id INTEGER,
-    destination VARCHAR(150),
-    pdu_id VARCHAR(150),
-    pdu_origin VARCHAR(150),
+    destination TEXT,
+    pdu_id TEXT,
+    pdu_origin TEXT,
     UNIQUE (transaction_id, destination)
 );
 
@@ -57,7 +57,7 @@ CREATE INDEX transaction_id_to_pdu_dest ON transaction_id_to_pdu(destination);
 
 -- To track destination health
 CREATE TABLE IF NOT EXISTS destinations(
-    destination VARCHAR(150) PRIMARY KEY,
+    destination TEXT PRIMARY KEY,
     retry_last_ts BIGINT,
     retry_interval INTEGER
 );
diff --git a/synapse/storage/schema/full_schemas/16/users.sql b/synapse/storage/schema/full_schemas/16/users.sql
index 033e3244b5..d2fa3122da 100644
--- a/synapse/storage/schema/full_schemas/16/users.sql
+++ b/synapse/storage/schema/full_schemas/16/users.sql
@@ -13,8 +13,8 @@
  * limitations under the License.
  */
 CREATE TABLE IF NOT EXISTS users(
-    name VARCHAR(150),
-    password_hash VARCHAR(150),
+    name TEXT,
+    password_hash TEXT,
     creation_ts BIGINT,
     admin SMALLINT DEFAULT 0 NOT NULL,
     UNIQUE(name)
@@ -22,18 +22,18 @@ CREATE TABLE IF NOT EXISTS users(
 
 CREATE TABLE IF NOT EXISTS access_tokens(
     id BIGINT PRIMARY KEY,
-    user_id VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    token VARCHAR(150) NOT NULL,
+    user_id TEXT NOT NULL,
+    device_id TEXT,
+    token TEXT NOT NULL,
     last_used BIGINT,
     UNIQUE(token)
 );
 
 CREATE TABLE IF NOT EXISTS user_ips (
-    user_id VARCHAR(150) NOT NULL,
-    access_token VARCHAR(150) NOT NULL,
-    device_id VARCHAR(150),
-    ip VARCHAR(150) NOT NULL,
+    user_id TEXT NOT NULL,
+    access_token TEXT NOT NULL,
+    device_id TEXT,
+    ip TEXT NOT NULL,
     user_agent TEXT NOT NULL,
     last_seen BIGINT NOT NULL
 );
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/schema_version.sql
index d9494611e0..d682608aa0 100644
--- a/synapse/storage/schema/schema_version.sql
+++ b/synapse/storage/schema/schema_version.sql
@@ -22,6 +22,6 @@ CREATE TABLE IF NOT EXISTS schema_version(
 
 CREATE TABLE IF NOT EXISTS applied_schema_deltas(
     version INTEGER NOT NULL,
-    file VARCHAR(150) NOT NULL,
+    file TEXT NOT NULL,
     UNIQUE(version, file)
 );
-- 
cgit 1.5.1